In [1]:
import sys
import platform
if platform.system() == 'Linux':
sys.path.append('/home/huiying/Dropbox/Program/')
elif platform.system() == 'Windows':
sys.path.append('D:/Dropbox/Program/')
from own_packages.SN import *
from own_packages.preprocess import *
from own_packages.Feature_Selection import *
from own_packages.Final_Model import *
import os
from numpy import *
from pandas import *
import matplotlib
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier as RF
%matplotlib inline
In [25]:
files = os.listdir()
# read the label(y)
y_all = read_csv('y_all.csv',header=0,index_col=0)
y_all = y_all.label
print(y_all)
# generate or read the No. list of cross-validation
if 'OneCV_ID_Test_CV.csv' in files and 'OneCV_ID_Train_CV.csv' in files:
ID_train = read_csv('OneCV_ID_Train_CV.csv',header=0,index_col=0)
ID_test = read_csv('OneCV_ID_Test_CV.csv',header=0,index_col=0)
else:
ID_train,ID_test = get_CVID_CSV(y_all,n_splits=5,name='OneCV')
# set the default parameters of plot
plot_default()
# set the algorithms
lr = LogisticRegression(solver='liblinear')
svc = svm.SVC(probability=True,gamma='scale')
rf = RF(n_estimators=100)
MLID
1234026 0
1290264 0
1299628 0
1313585 1
1319491 0
1319857 1
1340213 0
1345116 1
1347742 0
1360543 0
1361111 0
1363165 0
1404820 1
1412799 0
1422362 1
1474840 0
1475214 0
1476232 0
1482287 0
1487216 0
1494519 0
1503424 0
1505342 0
1506433 0
1513571 0
1516688 1
1520407 1
1530123 0
1571633 0
1584572 0
..
2588525 1
2592990 0
2594215 1
2610058 0
2628972 0
2642645 0
2685459 0
2693184 0
2712262 1
2722506 1
2809900 1
2825428 0
2832556 1
2845748 1
2847171 1
2872625 0
2885862 0
2916823 0
2935369 0
2944076 1
2945181 0
2947334 1
3014537 1
3029304 1
3062868 1
3067890 0
3093550 1
3100166 1
3105376 0
3113885 1
Name: label, Length: 124, dtype: int64
In [28]:
# set the name of dirs and names
fpath_out = 'T2'
name = 'T2'
# read and standardize the X
if 'T2' in files:
X_T2_Sd = read_csv('T2/X_T2_Sd.csv',header=0,index_col=0)
else:
if not os.path.exists(fpath_out):os.makedirs(fpath_out)
X_T2 = read_csv('X_all_T2.csv',header=0,index_col=0)
# X_T2_Sd = Std_features(X_T2)
# X_T2_Sd.to_csv('T2/X_T2_Sd.csv')
# set the structure of inputs
fpath_out = 'T2/LASSO'
if not os.path.exists(fpath_out):os.makedirs(fpath_out)
T2_input = paras(X=X_T2,y=y_all,ID_test=ID_test,ID_train=ID_train,
name=name,fpath=fpath_out)
# the first step of feature selection
T2_output_Lasso = LASSO_CV.selection(T2_input,inner_n_splits=10)
plt.close('all')
# model developing
T2_output_Further = T2_output_Lasso
for clf,model_name in [(lr,'LR'),(svc,'SVM'),(rf,'RF')]:
T2_output_Lasso.model = clf
T2_output_Lasso.name = model_name
T2_output_Lasso.fpath = fpath_out
T2_results_CV,T2_result_all = ModelPred.OneCV(T2_output_Lasso,label=['Neg','Pos'])
plt.close()
-----Start LASSO_CV----- Alpha: 0.07374151122277184
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Features reduced from 1029 to 6 by LASSO
Coefficients
Features
wavelet-HHL_firstorder_Median -0.080493
wavelet-LHL_firstorder_Kurtosis -0.008507
wavelet-LLL_glcm_Imc1 0.001751
wavelet-LHL_firstorder_Skewness 0.007827
original_shape_MinorAxis 0.013365
original_shape_SurfaceArea 0.134488
-----0-----
Alpha: 0.07999008066126778
Features reduced from 1029 to 5 by LASSO
Coefficients
Features
wavelet-HHL_firstorder_Median -0.051056
wavelet-LHL_firstorder_Kurtosis -0.009830
original_shape_Compactness2 -0.005178
original_shape_MinorAxis 0.013909
original_shape_SurfaceArea 0.154586
-----1-----
Alpha: 0.0823511334487696
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Features reduced from 1029 to 5 by LASSO
Coefficients
Features
wavelet-HHL_firstorder_Median -0.058846
exponential_glszm_GrayLevelNonUniformityNormalized 0.006572
wavelet-HHL_glszm_LargeAreaHighGrayLevelEmphasis 0.009509
original_shape_MinorAxis 0.018076
original_shape_SurfaceArea 0.138468
-----2-----
Alpha: 0.10937061133294382
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Features reduced from 1029 to 3 by LASSO
Coefficients
Features
wavelet-HHL_firstorder_Median -0.004822
original_shape_MinorAxis 0.026952
original_shape_SurfaceArea 0.085461
-----3-----
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Alpha: 0.06076732428711335
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Features reduced from 1029 to 9 by LASSO
Coefficients
Features
wavelet-HHL_firstorder_Median -0.071032
original_shape_Compactness2 -0.013200
wavelet-HLH_firstorder_Skewness -0.004270
wavelet-HLL_firstorder_Skewness 0.004617
wavelet-HHL_glszm_LargeAreaHighGrayLevelEmphasis 0.010977
original_glszm_SmallAreaLowGrayLevelEmphasis 0.012407
wavelet-HHL_glrlm_RunVariance 0.025580
wavelet-LHL_firstorder_Skewness 0.033874
original_shape_SurfaceArea 0.165685
-----4-----
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
Neg 0.88 1.00 0.93 14
Pos 1.00 0.82 0.90 11
micro avg 0.92 0.92 0.92 25
macro avg 0.94 0.91 0.92 25
weighted avg 0.93 0.92 0.92 25
[Confusion Matrix]
TN FP
FN TP
[[14 0]
[ 2 9]]
cutoff = ('Youden', 0.4267072366789403)
recall = 0.8181818181818182
precision = 1.0
sensitivity = 0.8181818181818182
specificity = 1.0
accuracy = 0.92
F1 = 0.9
brier = 0.14225119104950892
AUC = 0.922077922077922
95%CI-AUC = ('Binomial', 0.7425027172678562, 0.9908159374803416)
------train------
precision recall f1-score support
Neg 0.88 0.82 0.85 56
Pos 0.79 0.86 0.82 43
micro avg 0.84 0.84 0.84 99
macro avg 0.84 0.84 0.84 99
weighted avg 0.84 0.84 0.84 99
[Confusion Matrix]
TN FP
FN TP
[[46 10]
[ 6 37]]
cutoff = ('Youden', 0.5219271690291363)
recall = 0.8604651162790697
precision = 0.7872340425531915
sensitivity = 0.8604651162790697
specificity = 0.8214285714285714
accuracy = 0.8383838383838383
F1 = 0.8222222222222222
brier = 0.13948373768088151
AUC = 0.880813953488372
95%CI-AUC = ('Binomial', 0.8002426948579309, 0.937276419220511)
-----1-----
------test------
precision recall f1-score support
Neg 1.00 0.57 0.73 14
Pos 0.65 1.00 0.79 11
micro avg 0.76 0.76 0.76 25
macro avg 0.82 0.79 0.76 25
weighted avg 0.84 0.76 0.75 25
[Confusion Matrix]
TN FP
FN TP
[[ 8 6]
[ 0 11]]
cutoff = ('Youden', 0.42278606692802445)
recall = 1.0
precision = 0.6470588235294118
sensitivity = 1.0
specificity = 0.5714285714285714
accuracy = 0.76
F1 = 0.7857142857142858
brier = 0.21611177912884738
AUC = 0.7207792207792207
95%CI-AUC = ('Binomial', 0.506938318533891, 0.8798285008906752)
------train------
precision recall f1-score support
Neg 0.92 0.84 0.88 56
Pos 0.81 0.91 0.86 43
micro avg 0.87 0.87 0.87 99
macro avg 0.87 0.87 0.87 99
weighted avg 0.87 0.87 0.87 99
[Confusion Matrix]
TN FP
FN TP
[[47 9]
[ 4 39]]
cutoff = ('Youden', 0.434834054339196)
recall = 0.9069767441860465
precision = 0.8125
sensitivity = 0.9069767441860465
specificity = 0.8392857142857143
accuracy = 0.8686868686868687
F1 = 0.8571428571428572
brier = 0.12520390307881762
AUC = 0.9003322259136213
95%CI-AUC = ('Binomial', 0.8237075049706944, 0.9514430078931789)
-----2-----
------test------
precision recall f1-score support
Neg 0.82 0.64 0.72 14
Pos 0.64 0.82 0.72 11
micro avg 0.72 0.72 0.72 25
macro avg 0.73 0.73 0.72 25
weighted avg 0.74 0.72 0.72 25
[Confusion Matrix]
TN FP
FN TP
[[9 5]
[2 9]]
cutoff = ('Youden', 0.4408220182667646)
recall = 0.8181818181818182
precision = 0.6428571428571429
sensitivity = 0.8181818181818182
specificity = 0.6428571428571429
accuracy = 0.72
F1 = 0.7200000000000001
brier = 0.21619411310923273
AUC = 0.7142857142857143
95%CI-AUC = ('Binomial', 0.5001623351600223, 0.8752665811796557)
------train------
precision recall f1-score support
Neg 0.92 0.80 0.86 56
Pos 0.78 0.91 0.84 43
micro avg 0.85 0.85 0.85 99
macro avg 0.85 0.86 0.85 99
weighted avg 0.86 0.85 0.85 99
[Confusion Matrix]
TN FP
FN TP
[[45 11]
[ 4 39]]
cutoff = ('Youden', 0.4224025188527173)
recall = 0.9069767441860465
precision = 0.78
sensitivity = 0.9069767441860465
specificity = 0.8035714285714286
accuracy = 0.8484848484848485
F1 = 0.8387096774193548
brier = 0.12760580917330744
AUC = 0.8982558139534883
95%CI-AUC = ('Binomial', 0.8211841253135906, 0.9499661051802076)
-----3-----
------test------
precision recall f1-score support
Neg 1.00 0.79 0.88 14
Pos 0.79 1.00 0.88 11
micro avg 0.88 0.88 0.88 25
macro avg 0.89 0.89 0.88 25
weighted avg 0.91 0.88 0.88 25
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 0 11]]
cutoff = ('Youden', 0.33555088208194583)
recall = 1.0
precision = 0.7857142857142857
sensitivity = 1.0
specificity = 0.7857142857142857
accuracy = 0.88
F1 = 0.88
brier = 0.12476560348807517
AUC = 0.9545454545454546
95%CI-AUC = ('Binomial', 0.7883419356554456, 0.9983179285128623)
------train------
precision recall f1-score support
Neg 0.82 0.84 0.83 56
Pos 0.79 0.77 0.78 43
micro avg 0.81 0.81 0.81 99
macro avg 0.81 0.80 0.80 99
weighted avg 0.81 0.81 0.81 99
[Confusion Matrix]
TN FP
FN TP
[[47 9]
[10 33]]
cutoff = ('Youden', 0.491060078753171)
recall = 0.7674418604651163
precision = 0.7857142857142857
sensitivity = 0.7674418604651163
specificity = 0.8392857142857143
accuracy = 0.8080808080808081
F1 = 0.7764705882352941
brier = 0.16721251866738704
AUC = 0.8251661129568106
95%CI-AUC = ('Binomial', 0.7358328768495677, 0.8941670790806746)
-----4-----
------test------
precision recall f1-score support
Neg 0.82 0.64 0.72 14
Pos 0.62 0.80 0.70 10
micro avg 0.71 0.71 0.71 24
macro avg 0.72 0.72 0.71 24
weighted avg 0.73 0.71 0.71 24
[Confusion Matrix]
TN FP
FN TP
[[9 5]
[2 8]]
cutoff = ('Youden', 0.43066135782392717)
recall = 0.8
precision = 0.6153846153846154
sensitivity = 0.8
specificity = 0.6428571428571429
accuracy = 0.7083333333333334
F1 = 0.6956521739130435
brier = 0.24367684314974705
AUC = 0.7285714285714285
95%CI-AUC = ('Binomial', 0.510137636928788, 0.8878833127936933)
------train------
precision recall f1-score support
Neg 0.94 0.79 0.85 56
Pos 0.77 0.93 0.85 44
micro avg 0.85 0.85 0.85 100
macro avg 0.85 0.86 0.85 100
weighted avg 0.86 0.85 0.85 100
[Confusion Matrix]
TN FP
FN TP
[[44 12]
[ 3 41]]
cutoff = ('Youden', 0.39405096960460056)
recall = 0.9318181818181818
precision = 0.7735849056603774
sensitivity = 0.9318181818181818
specificity = 0.7857142857142857
accuracy = 0.85
F1 = 0.8453608247422681
brier = 0.12052680266926385
AUC = 0.9119318181818182
95%CI-AUC = ('Binomial', 0.8384028015162921, 0.9593496496395022)
0 1 2 3 4 Mean \
recall 0.860465 0.906977 0.906977 0.767442 0.931818 0.874736
precision 0.787234 0.812500 0.780000 0.785714 0.773585 0.787807
sensitivity 0.860465 0.906977 0.906977 0.767442 0.931818 0.874736
specificity 0.821429 0.839286 0.803571 0.839286 0.785714 0.817857
accuracy 0.838384 0.868687 0.848485 0.808081 0.850000 0.842727
F1 0.822222 0.857143 0.838710 0.776471 0.845361 0.827981
brier 0.139484 0.125204 0.127606 0.167213 0.120527 0.136007
AUC 0.880814 0.900332 0.898256 0.825166 0.911932 0.883300
Std
recall 0.065294
precision 0.014813
sensitivity 0.065294
specificity 0.023283
accuracy 0.022244
F1 0.031441
brier 0.018794
AUC 0.034349
0 1 2 3 4 Mean \
recall 0.818182 1.000000 0.818182 1.000000 0.800000 0.887273
precision 1.000000 0.647059 0.642857 0.785714 0.615385 0.738203
sensitivity 0.818182 1.000000 0.818182 1.000000 0.800000 0.887273
specificity 1.000000 0.571429 0.642857 0.785714 0.642857 0.728571
accuracy 0.920000 0.760000 0.720000 0.880000 0.708333 0.797667
F1 0.900000 0.785714 0.720000 0.880000 0.695652 0.796273
brier 0.142251 0.216112 0.216194 0.124766 0.243677 0.188600
AUC 0.922078 0.720779 0.714286 0.954545 0.728571 0.808052
Std
recall 0.103173
precision 0.160684
sensitivity 0.103173
specificity 0.170533
accuracy 0.096405
F1 0.091955
brier 0.051901
AUC 0.119570
------all------
precision recall f1-score support
Neg 0.87 0.74 0.80 70
Pos 0.72 0.85 0.78 54
micro avg 0.79 0.79 0.79 124
macro avg 0.79 0.80 0.79 124
weighted avg 0.80 0.79 0.79 124
[Confusion Matrix]
TN FP
FN TP
[[52 18]
[ 8 46]]
cutoff = ('Youden', 0.41835055833140555)
recall = 0.8518518518518519
precision = 0.71875
sensitivity = 0.8518518518518519
specificity = 0.7428571428571429
accuracy = 0.7903225806451613
F1 = 0.7796610169491525
brier = 0.1881557371369801
AUC = 0.7891534391534393
95%CI-AUC = ('Binomial', 0.7067808101740262, 0.8572289148665777)
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
Neg 0.86 0.86 0.86 14
Pos 0.82 0.82 0.82 11
micro avg 0.84 0.84 0.84 25
macro avg 0.84 0.84 0.84 25
weighted avg 0.84 0.84 0.84 25
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 2 9]]
cutoff = ('Youden', 0.42616384289488374)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.17626465964922364
AUC = 0.7922077922077921
95%CI-AUC = ('Binomial', 0.5841998281075556, 0.9269351737428029)
------train------
precision recall f1-score support
Neg 0.96 0.79 0.86 56
Pos 0.77 0.95 0.85 43
micro avg 0.86 0.86 0.86 99
macro avg 0.87 0.87 0.86 99
weighted avg 0.88 0.86 0.86 99
[Confusion Matrix]
TN FP
FN TP
[[44 12]
[ 2 41]]
cutoff = ('Youden', 0.36059121670305017)
recall = 0.9534883720930233
precision = 0.7735849056603774
sensitivity = 0.9534883720930233
specificity = 0.7857142857142857
accuracy = 0.8585858585858586
F1 = 0.8541666666666667
brier = 0.1126683474014466
AUC = 0.9102990033222591
95%CI-AUC = ('Binomial', 0.8359207439334404, 0.958418504744027)
-----1-----
------test------
precision recall f1-score support
Neg 0.89 0.57 0.70 14
Pos 0.62 0.91 0.74 11
micro avg 0.72 0.72 0.72 25
macro avg 0.76 0.74 0.72 25
weighted avg 0.77 0.72 0.72 25
[Confusion Matrix]
TN FP
FN TP
[[ 8 6]
[ 1 10]]
cutoff = ('Youden', 0.5)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.20927435962451896
AUC = 0.7012987012987013
95%CI-AUC = ('Binomial', 0.4867234203121368, 0.8660179541627924)
------train------
precision recall f1-score support
Neg 0.84 0.96 0.90 56
Pos 0.94 0.77 0.85 43
micro avg 0.88 0.88 0.88 99
macro avg 0.89 0.87 0.87 99
weighted avg 0.89 0.88 0.88 99
[Confusion Matrix]
TN FP
FN TP
[[54 2]
[10 33]]
cutoff = ('Youden', 0.615372892842085)
recall = 0.7674418604651163
precision = 0.9428571428571428
sensitivity = 0.7674418604651163
specificity = 0.9642857142857143
accuracy = 0.8787878787878788
F1 = 0.8461538461538461
brier = 0.10293226916354008
AUC = 0.9227574750830565
95%CI-AUC = ('Binomial', 0.8514499690678909, 0.9668390113927129)
-----2-----
------test------
precision recall f1-score support
Neg 0.85 0.79 0.81 14
Pos 0.75 0.82 0.78 11
micro avg 0.80 0.80 0.80 25
macro avg 0.80 0.80 0.80 25
weighted avg 0.80 0.80 0.80 25
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 2 9]]
cutoff = ('Youden', 0.5184348239523857)
recall = 0.8181818181818182
precision = 0.75
sensitivity = 0.8181818181818182
specificity = 0.7857142857142857
accuracy = 0.8
F1 = 0.7826086956521738
brier = 0.17410291442452358
AUC = 0.8116883116883117
95%CI-AUC = ('Binomial', 0.6062459490380402, 0.9386536573797263)
------train------
precision recall f1-score support
Neg 0.89 0.88 0.88 56
Pos 0.84 0.86 0.85 43
micro avg 0.87 0.87 0.87 99
macro avg 0.87 0.87 0.87 99
weighted avg 0.87 0.87 0.87 99
[Confusion Matrix]
TN FP
FN TP
[[49 7]
[ 6 37]]
cutoff = ('Youden', 0.48787607002262756)
recall = 0.8604651162790697
precision = 0.8409090909090909
sensitivity = 0.8604651162790697
specificity = 0.875
accuracy = 0.8686868686868687
F1 = 0.8505747126436781
brier = 0.12070078306976685
AUC = 0.9028239202657807
95%CI-AUC = ('Binomial', 0.8267448381442899, 0.9532048874802718)
-----3-----
------test------
precision recall f1-score support
Neg 0.92 0.79 0.85 14
Pos 0.77 0.91 0.83 11
micro avg 0.84 0.84 0.84 25
macro avg 0.84 0.85 0.84 25
weighted avg 0.85 0.84 0.84 25
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 1 10]]
cutoff = ('Youden', 0.2534909236873359)
recall = 0.9090909090909091
precision = 0.7692307692307693
sensitivity = 0.9090909090909091
specificity = 0.7857142857142857
accuracy = 0.84
F1 = 0.8333333333333333
brier = 0.1350683289008123
AUC = 0.9285714285714285
95%CI-AUC = ('Binomial', 0.7513653244637356, 0.9927435353916194)
------train------
precision recall f1-score support
Neg 0.80 0.88 0.84 56
Pos 0.82 0.72 0.77 43
micro avg 0.81 0.81 0.81 99
macro avg 0.81 0.80 0.80 99
weighted avg 0.81 0.81 0.81 99
[Confusion Matrix]
TN FP
FN TP
[[49 7]
[12 31]]
cutoff = ('Youden', 0.5242508293427226)
recall = 0.7209302325581395
precision = 0.8157894736842105
sensitivity = 0.7209302325581395
specificity = 0.875
accuracy = 0.8080808080808081
F1 = 0.765432098765432
brier = 0.153285283164566
AUC = 0.8666943521594684
95%CI-AUC = ('Binomial', 0.7835864224034667, 0.926677498055638)
-----4-----
------test------
precision recall f1-score support
Neg 0.85 0.79 0.81 14
Pos 0.73 0.80 0.76 10
micro avg 0.79 0.79 0.79 24
macro avg 0.79 0.79 0.79 24
weighted avg 0.80 0.79 0.79 24
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 2 8]]
cutoff = ('Youden', 0.5)
recall = 0.8
precision = 0.7272727272727273
sensitivity = 0.8
specificity = 0.7857142857142857
accuracy = 0.7916666666666666
F1 = 0.761904761904762
brier = 0.18387271923771134
AUC = 0.7857142857142857
95%CI-AUC = ('Binomial', 0.5718546926357083, 0.9250456179687029)
------train------
precision recall f1-score support
Neg 0.93 0.91 0.92 56
Pos 0.89 0.91 0.90 44
micro avg 0.91 0.91 0.91 100
macro avg 0.91 0.91 0.91 100
weighted avg 0.91 0.91 0.91 100
[Confusion Matrix]
TN FP
FN TP
[[51 5]
[ 4 40]]
cutoff = ('Youden', 0.4836372877167067)
recall = 0.9090909090909091
precision = 0.8888888888888888
sensitivity = 0.9090909090909091
specificity = 0.9107142857142857
accuracy = 0.91
F1 = 0.8988764044943819
brier = 0.09107219405859732
AUC = 0.9594155844155844
95%CI-AUC = ('Binomial', 0.8999369465778991, 0.988694092758204)
0 1 2 3 4 Mean \
recall 0.953488 0.767442 0.860465 0.720930 0.909091 0.842283
precision 0.773585 0.942857 0.840909 0.815789 0.888889 0.852406
sensitivity 0.953488 0.767442 0.860465 0.720930 0.909091 0.842283
specificity 0.785714 0.964286 0.875000 0.875000 0.910714 0.882143
accuracy 0.858586 0.878788 0.868687 0.808081 0.910000 0.864828
F1 0.854167 0.846154 0.850575 0.765432 0.898876 0.843041
brier 0.112668 0.102932 0.120701 0.153285 0.091072 0.116132
AUC 0.910299 0.922757 0.902824 0.866694 0.959416 0.912398
Std
recall 0.096809
precision 0.065571
sensitivity 0.096809
specificity 0.065124
accuracy 0.037113
F1 0.048298
brier 0.023534
AUC 0.033552
0 1 2 3 4 Mean \
recall 0.818182 0.909091 0.818182 0.909091 0.800000 0.850909
precision 0.818182 0.625000 0.750000 0.769231 0.727273 0.737937
sensitivity 0.818182 0.909091 0.818182 0.909091 0.800000 0.850909
specificity 0.857143 0.571429 0.785714 0.785714 0.785714 0.757143
accuracy 0.840000 0.720000 0.800000 0.840000 0.791667 0.798333
F1 0.818182 0.740741 0.782609 0.833333 0.761905 0.787354
brier 0.176265 0.209274 0.174103 0.135068 0.183873 0.175717
AUC 0.792208 0.701299 0.811688 0.928571 0.785714 0.803896
Std
recall 0.053629
precision 0.071471
sensitivity 0.053629
specificity 0.108327
accuracy 0.049131
F1 0.038430
brier 0.026683
AUC 0.081545
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------all------
precision recall f1-score support
Neg 0.85 0.79 0.81 70
Pos 0.75 0.81 0.78 54
micro avg 0.80 0.80 0.80 124
macro avg 0.80 0.80 0.80 124
weighted avg 0.80 0.80 0.80 124
[Confusion Matrix]
TN FP
FN TP
[[55 15]
[10 44]]
cutoff = ('Youden', 0.42616384289488374)
recall = 0.8148148148148148
precision = 0.7457627118644068
sensitivity = 0.8148148148148148
specificity = 0.7857142857142857
accuracy = 0.7983870967741935
F1 = 0.7787610619469028
brier = 0.17565082118291966
AUC = 0.8034391534391534
95%CI-AUC = ('Binomial', 0.7225090298864401, 0.8693406224325474)
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
Neg 0.75 0.86 0.80 14
Pos 0.78 0.64 0.70 11
micro avg 0.76 0.76 0.76 25
macro avg 0.76 0.75 0.75 25
weighted avg 0.76 0.76 0.76 25
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 4 7]]
cutoff = ('Youden', 0.55)
recall = 0.6363636363636364
precision = 0.7777777777777778
sensitivity = 0.6363636363636364
specificity = 0.8571428571428571
accuracy = 0.76
F1 = 0.7000000000000001
brier = 0.20026
AUC = 0.737012987012987
95%CI-AUC = ('Binomial', 0.524048524156825, 0.8910439384661436)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
Neg 1.00 1.00 1.00 56
Pos 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.7)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.02496363636363637
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----1-----
------test------
precision recall f1-score support
Neg 0.83 0.71 0.77 14
Pos 0.69 0.82 0.75 11
micro avg 0.76 0.76 0.76 25
macro avg 0.76 0.77 0.76 25
weighted avg 0.77 0.76 0.76 25
[Confusion Matrix]
TN FP
FN TP
[[10 4]
[ 2 9]]
cutoff = ('Youden', 0.61)
recall = 0.8181818181818182
precision = 0.6923076923076923
sensitivity = 0.8181818181818182
specificity = 0.7142857142857143
accuracy = 0.76
F1 = 0.7500000000000001
brier = 0.17698400000000003
AUC = 0.827922077922078
95%CI-AUC = ('Binomial', 0.6249889329318279, 0.9479750440455293)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
Neg 1.00 1.00 1.00 56
Pos 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.62)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.026535353535353537
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----2-----
------test------
precision recall f1-score support
Neg 0.82 0.64 0.72 14
Pos 0.64 0.82 0.72 11
micro avg 0.72 0.72 0.72 25
macro avg 0.73 0.73 0.72 25
weighted avg 0.74 0.72 0.72 25
[Confusion Matrix]
TN FP
FN TP
[[9 5]
[2 9]]
cutoff = ('Youden', 0.36)
recall = 0.8181818181818182
precision = 0.6428571428571429
sensitivity = 0.8181818181818182
specificity = 0.6428571428571429
accuracy = 0.72
F1 = 0.7200000000000001
brier = 0.207364
AUC = 0.724025974025974
95%CI-AUC = ('Binomial', 0.5103407260113572, 0.8820934725837822)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
Neg 1.00 1.00 1.00 56
Pos 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.62)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.02494141414141414
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----3-----
------test------
precision recall f1-score support
Neg 0.92 0.79 0.85 14
Pos 0.77 0.91 0.83 11
micro avg 0.84 0.84 0.84 25
macro avg 0.84 0.85 0.84 25
weighted avg 0.85 0.84 0.84 25
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 1 10]]
cutoff = ('Youden', 0.29)
recall = 0.9090909090909091
precision = 0.7692307692307693
sensitivity = 0.9090909090909091
specificity = 0.7857142857142857
accuracy = 0.84
F1 = 0.8333333333333333
brier = 0.13916
AUC = 0.8733766233766235
95%CI-AUC = ('Binomial', 0.6795599341721734, 0.9714887603791876)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
Neg 1.00 1.00 1.00 56
Pos 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.58)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.028853535353535357
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----4-----
------test------
precision recall f1-score support
Neg 0.79 0.79 0.79 14
Pos 0.70 0.70 0.70 10
micro avg 0.75 0.75 0.75 24
macro avg 0.74 0.74 0.74 24
weighted avg 0.75 0.75 0.75 24
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 3 7]]
cutoff = ('Youden', 0.62)
recall = 0.7
precision = 0.7
sensitivity = 0.7
specificity = 0.7857142857142857
accuracy = 0.75
F1 = 0.7
brier = 0.211275
AUC = 0.7285714285714285
95%CI-AUC = ('Binomial', 0.510137636928788, 0.8878833127936933)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
Neg 1.00 1.00 1.00 56
Pos 1.00 1.00 1.00 44
micro avg 1.00 1.00 1.00 100
macro avg 1.00 1.00 1.00 100
weighted avg 1.00 1.00 1.00 100
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 44]]
cutoff = ('Youden', 0.66)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.024641000000000003
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9637833073548235, nan)
0 1 2 3 4 Mean \
recall 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
precision 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
sensitivity 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
specificity 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
accuracy 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
F1 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
brier 0.024964 0.026535 0.024941 0.028854 0.024641 0.025987
AUC 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Std
recall 0.000000
precision 0.000000
sensitivity 0.000000
specificity 0.000000
accuracy 0.000000
F1 0.000000
brier 0.001766
AUC 0.000000
0 1 2 3 4 Mean \
recall 0.636364 0.818182 0.818182 0.909091 0.700000 0.776364
precision 0.777778 0.692308 0.642857 0.769231 0.700000 0.716435
sensitivity 0.636364 0.818182 0.818182 0.909091 0.700000 0.776364
specificity 0.857143 0.714286 0.642857 0.785714 0.785714 0.757143
accuracy 0.760000 0.760000 0.720000 0.840000 0.750000 0.766000
F1 0.700000 0.750000 0.720000 0.833333 0.700000 0.740667
brier 0.200260 0.176984 0.207364 0.139160 0.211275 0.187009
AUC 0.737013 0.827922 0.724026 0.873377 0.728571 0.778182
Std
recall 0.107872
precision 0.056604
sensitivity 0.107872
specificity 0.081441
accuracy 0.044497
F1 0.055698
brier 0.029870
AUC 0.068237
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------all------
precision recall f1-score support
Neg 0.84 0.67 0.75 70
Pos 0.66 0.83 0.74 54
micro avg 0.74 0.74 0.74 124
macro avg 0.75 0.75 0.74 124
weighted avg 0.76 0.74 0.74 124
[Confusion Matrix]
TN FP
FN TP
[[47 23]
[ 9 45]]
cutoff = ('Youden', 0.36)
recall = 0.8333333333333334
precision = 0.6617647058823529
sensitivity = 0.8333333333333334
specificity = 0.6714285714285714
accuracy = 0.7419354838709677
F1 = 0.7377049180327869
brier = 0.18681290322580646
AUC = 0.7804232804232805
95%CI-AUC = ('Binomial', 0.6972318653162309, 0.849762074323568)
-----Finish!-----
In [29]:
# set the structure of inputs for further feature selection
fpath_out = 'T2/Ranking-5'
if not os.path.exists(fpath_out):os.makedirs(fpath_out)
T2_output_Lasso.typeof = Further_Selection.Ranking
T2_output_Lasso.model = lr
T2_output_Lasso.name = name
T2_output_Lasso.fpath = fpath_out
# the further feature selection
T2_output_Further = Further_Selection_CV.selection(T2_output_Lasso,max_FeaNum=5)
plt.close('all')
# model developing
for clf,model_name in [(lr,'LR'),(svc,'SVM'),(rf,'RF')]:
T2_output_Further.model = clf
T2_output_Further.name = model_name
T2_output_Further.fpath = fpath_out
T2_results_CV,T2_result_all = ModelPred.OneCV(T2_output_Further,label=['Neg','Pos'])
plt.close()
-----Start Further_Selection_CV-----
-----0-----
-----1-----
-----2-----
-----3-----
-----4-----
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
Neg 0.87 0.93 0.90 14
Pos 0.90 0.82 0.86 11
micro avg 0.88 0.88 0.88 25
macro avg 0.88 0.87 0.88 25
weighted avg 0.88 0.88 0.88 25
[Confusion Matrix]
TN FP
FN TP
[[13 1]
[ 2 9]]
cutoff = ('Youden', 0.42402832159446996)
recall = 0.8181818181818182
precision = 0.9
sensitivity = 0.8181818181818182
specificity = 0.9285714285714286
accuracy = 0.88
F1 = 0.8571428571428572
brier = 0.13747980475852128
AUC = 0.9285714285714286
95%CI-AUC = ('Binomial', 0.7513653244637356, 0.9927435353916194)
------train------
precision recall f1-score support
Neg 0.91 0.77 0.83 56
Pos 0.75 0.91 0.82 43
micro avg 0.83 0.83 0.83 99
macro avg 0.83 0.84 0.83 99
weighted avg 0.84 0.83 0.83 99
[Confusion Matrix]
TN FP
FN TP
[[43 13]
[ 4 39]]
cutoff = ('Youden', 0.4061461171253337)
recall = 0.9069767441860465
precision = 0.75
sensitivity = 0.9069767441860465
specificity = 0.7678571428571429
accuracy = 0.8282828282828283
F1 = 0.8210526315789475
brier = 0.1426153682210238
AUC = 0.8683554817275747
95%CI-AUC = ('Binomial', 0.7855335392847961, 0.9279380169555842)
-----1-----
------test------
precision recall f1-score support
Neg 1.00 0.57 0.73 14
Pos 0.65 1.00 0.79 11
micro avg 0.76 0.76 0.76 25
macro avg 0.82 0.79 0.76 25
weighted avg 0.84 0.76 0.75 25
[Confusion Matrix]
TN FP
FN TP
[[ 8 6]
[ 0 11]]
cutoff = ('Youden', 0.42278606692802445)
recall = 1.0
precision = 0.6470588235294118
sensitivity = 1.0
specificity = 0.5714285714285714
accuracy = 0.76
F1 = 0.7857142857142858
brier = 0.21611177912884738
AUC = 0.7207792207792207
95%CI-AUC = ('Binomial', 0.506938318533891, 0.8798285008906752)
------train------
precision recall f1-score support
Neg 0.92 0.84 0.88 56
Pos 0.81 0.91 0.86 43
micro avg 0.87 0.87 0.87 99
macro avg 0.87 0.87 0.87 99
weighted avg 0.87 0.87 0.87 99
[Confusion Matrix]
TN FP
FN TP
[[47 9]
[ 4 39]]
cutoff = ('Youden', 0.43483405433919586)
recall = 0.9069767441860465
precision = 0.8125
sensitivity = 0.9069767441860465
specificity = 0.8392857142857143
accuracy = 0.8686868686868687
F1 = 0.8571428571428572
brier = 0.12520390307881765
AUC = 0.9003322259136213
95%CI-AUC = ('Binomial', 0.8237075049706944, 0.9514430078931789)
-----2-----
------test------
precision recall f1-score support
Neg 0.82 0.64 0.72 14
Pos 0.64 0.82 0.72 11
micro avg 0.72 0.72 0.72 25
macro avg 0.73 0.73 0.72 25
weighted avg 0.74 0.72 0.72 25
[Confusion Matrix]
TN FP
FN TP
[[9 5]
[2 9]]
cutoff = ('Youden', 0.4408220182667645)
recall = 0.8181818181818182
precision = 0.6428571428571429
sensitivity = 0.8181818181818182
specificity = 0.6428571428571429
accuracy = 0.72
F1 = 0.7200000000000001
brier = 0.21619411310923276
AUC = 0.7142857142857143
95%CI-AUC = ('Binomial', 0.5001623351600223, 0.8752665811796557)
------train------
precision recall f1-score support
Neg 0.92 0.80 0.86 56
Pos 0.78 0.91 0.84 43
micro avg 0.85 0.85 0.85 99
macro avg 0.85 0.86 0.85 99
weighted avg 0.86 0.85 0.85 99
[Confusion Matrix]
TN FP
FN TP
[[45 11]
[ 4 39]]
cutoff = ('Youden', 0.42240251885271723)
recall = 0.9069767441860465
precision = 0.78
sensitivity = 0.9069767441860465
specificity = 0.8035714285714286
accuracy = 0.8484848484848485
F1 = 0.8387096774193548
brier = 0.12760580917330744
AUC = 0.8982558139534883
95%CI-AUC = ('Binomial', 0.8211841253135906, 0.9499661051802076)
-----3-----
------test------
precision recall f1-score support
Neg 1.00 0.79 0.88 14
Pos 0.79 1.00 0.88 11
micro avg 0.88 0.88 0.88 25
macro avg 0.89 0.89 0.88 25
weighted avg 0.91 0.88 0.88 25
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 0 11]]
cutoff = ('Youden', 0.33555088208194583)
recall = 1.0
precision = 0.7857142857142857
sensitivity = 1.0
specificity = 0.7857142857142857
accuracy = 0.88
F1 = 0.88
brier = 0.12476560348807517
AUC = 0.9545454545454546
95%CI-AUC = ('Binomial', 0.7883419356554456, 0.9983179285128623)
------train------
precision recall f1-score support
Neg 0.82 0.84 0.83 56
Pos 0.79 0.77 0.78 43
micro avg 0.81 0.81 0.81 99
macro avg 0.81 0.80 0.80 99
weighted avg 0.81 0.81 0.81 99
[Confusion Matrix]
TN FP
FN TP
[[47 9]
[10 33]]
cutoff = ('Youden', 0.491060078753171)
recall = 0.7674418604651163
precision = 0.7857142857142857
sensitivity = 0.7674418604651163
specificity = 0.8392857142857143
accuracy = 0.8080808080808081
F1 = 0.7764705882352941
brier = 0.16721251866738704
AUC = 0.8251661129568106
95%CI-AUC = ('Binomial', 0.7358328768495677, 0.8941670790806746)
-----4-----
------test------
precision recall f1-score support
Neg 0.89 0.57 0.70 14
Pos 0.60 0.90 0.72 10
micro avg 0.71 0.71 0.71 24
macro avg 0.74 0.74 0.71 24
weighted avg 0.77 0.71 0.71 24
[Confusion Matrix]
TN FP
FN TP
[[8 6]
[1 9]]
cutoff = ('Youden', 0.22255149324338114)
recall = 0.9
precision = 0.6
sensitivity = 0.9
specificity = 0.5714285714285714
accuracy = 0.7083333333333334
F1 = 0.7200000000000001
brier = 0.21498134261464819
AUC = 0.75
95%CI-AUC = ('Binomial', 0.5328871975773306, 0.9022695905254671)
------train------
precision recall f1-score support
Neg 0.92 0.82 0.87 56
Pos 0.80 0.91 0.85 44
micro avg 0.86 0.86 0.86 100
macro avg 0.86 0.87 0.86 100
weighted avg 0.87 0.86 0.86 100
[Confusion Matrix]
TN FP
FN TP
[[46 10]
[ 4 40]]
cutoff = ('Youden', 0.4201472105200088)
recall = 0.9090909090909091
precision = 0.8
sensitivity = 0.9090909090909091
specificity = 0.8214285714285714
accuracy = 0.86
F1 = 0.8510638297872342
brier = 0.1307433166281341
AUC = 0.8944805194805194
95%CI-AUC = ('Binomial', 0.8170919016549374, 0.9470409031339475)
0 1 2 3 4 Mean \
recall 0.906977 0.906977 0.906977 0.767442 0.909091 0.879493
precision 0.750000 0.812500 0.780000 0.785714 0.800000 0.785643
sensitivity 0.906977 0.906977 0.906977 0.767442 0.909091 0.879493
specificity 0.767857 0.839286 0.803571 0.839286 0.821429 0.814286
accuracy 0.828283 0.868687 0.848485 0.808081 0.860000 0.842707
F1 0.821053 0.857143 0.838710 0.776471 0.851064 0.828888
brier 0.142615 0.125204 0.127606 0.167213 0.130743 0.138676
AUC 0.868355 0.900332 0.898256 0.825166 0.894481 0.877318
Std
recall 0.062645
precision 0.023610
sensitivity 0.062645
specificity 0.029881
accuracy 0.024568
F1 0.032387
brier 0.017298
AUC 0.031870
0 1 2 3 4 Mean \
recall 0.818182 1.000000 0.818182 1.000000 0.900000 0.907273
precision 0.900000 0.647059 0.642857 0.785714 0.600000 0.715126
sensitivity 0.818182 1.000000 0.818182 1.000000 0.900000 0.907273
specificity 0.928571 0.571429 0.642857 0.785714 0.571429 0.700000
accuracy 0.880000 0.760000 0.720000 0.880000 0.708333 0.789667
F1 0.857143 0.785714 0.720000 0.880000 0.720000 0.792571
brier 0.137480 0.216112 0.216194 0.124766 0.214981 0.181907
AUC 0.928571 0.720779 0.714286 0.954545 0.750000 0.813636
Std
recall 0.091000
precision 0.124771
sensitivity 0.091000
specificity 0.154853
accuracy 0.084659
F1 0.074822
brier 0.046579
AUC 0.117907
------all------
precision recall f1-score support
Neg 0.88 0.71 0.79 70
Pos 0.70 0.87 0.78 54
micro avg 0.78 0.78 0.78 124
macro avg 0.79 0.79 0.78 124
weighted avg 0.80 0.78 0.78 124
[Confusion Matrix]
TN FP
FN TP
[[50 20]
[ 7 47]]
cutoff = ('Youden', 0.39294575016447075)
recall = 0.8703703703703703
precision = 0.7014925373134329
sensitivity = 0.8703703703703703
specificity = 0.7142857142857143
accuracy = 0.782258064516129
F1 = 0.7768595041322315
brier = 0.1816397962489393
AUC = 0.8007936507936508
95%CI-AUC = ('Binomial', 0.7195864395976709, 0.8671080854131452)
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
Neg 0.86 0.86 0.86 14
Pos 0.82 0.82 0.82 11
micro avg 0.84 0.84 0.84 25
macro avg 0.84 0.84 0.84 25
weighted avg 0.84 0.84 0.84 25
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 2 9]]
cutoff = ('Youden', 0.4630879768356677)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.16489748780253388
AUC = 0.8116883116883117
95%CI-AUC = ('Binomial', 0.6062459490380403, 0.9386536573797263)
------train------
precision recall f1-score support
Neg 0.96 0.77 0.85 56
Pos 0.76 0.95 0.85 43
micro avg 0.85 0.85 0.85 99
macro avg 0.86 0.86 0.85 99
weighted avg 0.87 0.85 0.85 99
[Confusion Matrix]
TN FP
FN TP
[[43 13]
[ 2 41]]
cutoff = ('Youden', 0.2774553422787947)
recall = 0.9534883720930233
precision = 0.7592592592592593
sensitivity = 0.9534883720930233
specificity = 0.7678571428571429
accuracy = 0.8484848484848485
F1 = 0.845360824742268
brier = 0.11996114772641048
AUC = 0.8978405315614617
95%CI-AUC = ('Binomial', 0.8206802758207263, 0.9496697996819241)
-----1-----
------test------
precision recall f1-score support
Neg 0.89 0.57 0.70 14
Pos 0.62 0.91 0.74 11
micro avg 0.72 0.72 0.72 25
macro avg 0.76 0.74 0.72 25
weighted avg 0.77 0.72 0.72 25
[Confusion Matrix]
TN FP
FN TP
[[ 8 6]
[ 1 10]]
cutoff = ('Youden', 0.4800329057500271)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.2068270380367323
AUC = 0.7012987012987013
95%CI-AUC = ('Binomial', 0.4867234203121368, 0.8660179541627924)
------train------
precision recall f1-score support
Neg 0.84 0.96 0.90 56
Pos 0.94 0.77 0.85 43
micro avg 0.88 0.88 0.88 99
macro avg 0.89 0.87 0.87 99
weighted avg 0.89 0.88 0.88 99
[Confusion Matrix]
TN FP
FN TP
[[54 2]
[10 33]]
cutoff = ('Youden', 0.5885786575201262)
recall = 0.7674418604651163
precision = 0.9428571428571428
sensitivity = 0.7674418604651163
specificity = 0.9642857142857143
accuracy = 0.8787878787878788
F1 = 0.8461538461538461
brier = 0.10544300296348887
AUC = 0.9227574750830565
95%CI-AUC = ('Binomial', 0.8514499690678909, 0.9668390113927129)
-----2-----
------test------
precision recall f1-score support
Neg 0.85 0.79 0.81 14
Pos 0.75 0.82 0.78 11
micro avg 0.80 0.80 0.80 25
macro avg 0.80 0.80 0.80 25
weighted avg 0.80 0.80 0.80 25
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 2 9]]
cutoff = ('Youden', 0.5339249544804243)
recall = 0.8181818181818182
precision = 0.75
sensitivity = 0.8181818181818182
specificity = 0.7857142857142857
accuracy = 0.8
F1 = 0.7826086956521738
brier = 0.17430344559520322
AUC = 0.8116883116883117
95%CI-AUC = ('Binomial', 0.6062459490380402, 0.9386536573797263)
------train------
precision recall f1-score support
Neg 0.89 0.88 0.88 56
Pos 0.84 0.86 0.85 43
micro avg 0.87 0.87 0.87 99
macro avg 0.87 0.87 0.87 99
weighted avg 0.87 0.87 0.87 99
[Confusion Matrix]
TN FP
FN TP
[[49 7]
[ 6 37]]
cutoff = ('Youden', 0.5)
recall = 0.8604651162790697
precision = 0.8409090909090909
sensitivity = 0.8604651162790697
specificity = 0.875
accuracy = 0.8686868686868687
F1 = 0.8505747126436781
brier = 0.12134654550691594
AUC = 0.9028239202657807
95%CI-AUC = ('Binomial', 0.8267448381442899, 0.9532048874802718)
-----3-----
------test------
precision recall f1-score support
Neg 0.92 0.79 0.85 14
Pos 0.77 0.91 0.83 11
micro avg 0.84 0.84 0.84 25
macro avg 0.84 0.85 0.84 25
weighted avg 0.85 0.84 0.84 25
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 1 10]]
cutoff = ('Youden', 0.26034276542559837)
recall = 0.9090909090909091
precision = 0.7692307692307693
sensitivity = 0.9090909090909091
specificity = 0.7857142857142857
accuracy = 0.84
F1 = 0.8333333333333333
brier = 0.1357103317659301
AUC = 0.9285714285714285
95%CI-AUC = ('Binomial', 0.7513653244637356, 0.9927435353916194)
------train------
precision recall f1-score support
Neg 0.80 0.88 0.84 56
Pos 0.82 0.72 0.77 43
micro avg 0.81 0.81 0.81 99
macro avg 0.81 0.80 0.80 99
weighted avg 0.81 0.81 0.81 99
[Confusion Matrix]
TN FP
FN TP
[[49 7]
[12 31]]
cutoff = ('Youden', 0.5253459652307046)
recall = 0.7209302325581395
precision = 0.8157894736842105
sensitivity = 0.7209302325581395
specificity = 0.875
accuracy = 0.8080808080808081
F1 = 0.765432098765432
brier = 0.15373927863701353
AUC = 0.8666943521594684
95%CI-AUC = ('Binomial', 0.7835864224034667, 0.926677498055638)
-----4-----
------test------
precision recall f1-score support
Neg 0.86 0.86 0.86 14
Pos 0.80 0.80 0.80 10
micro avg 0.83 0.83 0.83 24
macro avg 0.83 0.83 0.83 24
weighted avg 0.83 0.83 0.83 24
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 2 8]]
cutoff = ('Youden', 0.5242377306614807)
recall = 0.8
precision = 0.8
sensitivity = 0.8
specificity = 0.8571428571428571
accuracy = 0.8333333333333334
F1 = 0.8000000000000002
brier = 0.17083584456271447
AUC = 0.7928571428571429
95%CI-AUC = ('Binomial', 0.5798186525343071, 0.9294026470471334)
------train------
precision recall f1-score support
Neg 0.91 0.89 0.90 56
Pos 0.87 0.89 0.88 44
micro avg 0.89 0.89 0.89 100
macro avg 0.89 0.89 0.89 100
weighted avg 0.89 0.89 0.89 100
[Confusion Matrix]
TN FP
FN TP
[[50 6]
[ 5 39]]
cutoff = ('Youden', 0.44840286540223245)
recall = 0.8863636363636364
precision = 0.8666666666666667
sensitivity = 0.8863636363636364
specificity = 0.8928571428571429
accuracy = 0.89
F1 = 0.8764044943820225
brier = 0.09600555919715699
AUC = 0.9443993506493507
95%CI-AUC = ('Binomial', 0.8797335382271161, 0.9803126228456718)
0 1 2 3 4 Mean \
recall 0.953488 0.767442 0.860465 0.720930 0.886364 0.837738
precision 0.759259 0.942857 0.840909 0.815789 0.866667 0.845096
sensitivity 0.953488 0.767442 0.860465 0.720930 0.886364 0.837738
specificity 0.767857 0.964286 0.875000 0.875000 0.892857 0.875000
accuracy 0.848485 0.878788 0.868687 0.808081 0.890000 0.858808
F1 0.845361 0.846154 0.850575 0.765432 0.876404 0.836785
brier 0.119961 0.105443 0.121347 0.153739 0.096006 0.119299
AUC 0.897841 0.922757 0.902824 0.866694 0.944399 0.906903
Std
recall 0.093360
precision 0.067578
sensitivity 0.093360
specificity 0.070304
accuracy 0.032208
F1 0.041870
brier 0.021939
AUC 0.029038
0 1 2 3 4 Mean \
recall 0.818182 0.909091 0.818182 0.909091 0.800000 0.850909
precision 0.818182 0.625000 0.750000 0.769231 0.800000 0.752483
sensitivity 0.818182 0.909091 0.818182 0.909091 0.800000 0.850909
specificity 0.857143 0.571429 0.785714 0.785714 0.857143 0.771429
accuracy 0.840000 0.720000 0.800000 0.840000 0.833333 0.806667
F1 0.818182 0.740741 0.782609 0.833333 0.800000 0.794973
brier 0.164897 0.206827 0.174303 0.135710 0.170836 0.170515
AUC 0.811688 0.701299 0.811688 0.928571 0.792857 0.809221
Std
recall 0.053629
precision 0.076014
sensitivity 0.053629
specificity 0.117369
accuracy 0.051208
F1 0.035810
brier 0.025377
AUC 0.080888
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------all------
precision recall f1-score support
Neg 0.85 0.79 0.81 70
Pos 0.75 0.81 0.78 54
micro avg 0.80 0.80 0.80 124
macro avg 0.80 0.80 0.80 124
weighted avg 0.80 0.80 0.80 124
[Confusion Matrix]
TN FP
FN TP
[[55 15]
[10 44]]
cutoff = ('Youden', 0.43150875541595096)
recall = 0.8148148148148148
precision = 0.7457627118644068
sensitivity = 0.8148148148148148
specificity = 0.7857142857142857
accuracy = 0.7983870967741935
F1 = 0.7787610619469028
brier = 0.17051224072189622
AUC = 0.8115079365079365
95%CI-AUC = ('Binomial', 0.7314520381363178, 0.8761194755694358)
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
Neg 0.83 0.71 0.77 14
Pos 0.69 0.82 0.75 11
micro avg 0.76 0.76 0.76 25
macro avg 0.76 0.77 0.76 25
weighted avg 0.77 0.76 0.76 25
[Confusion Matrix]
TN FP
FN TP
[[10 4]
[ 2 9]]
cutoff = ('Youden', 0.45)
recall = 0.8181818181818182
precision = 0.6923076923076923
sensitivity = 0.8181818181818182
specificity = 0.7142857142857143
accuracy = 0.76
F1 = 0.7500000000000001
brier = 0.204808
AUC = 0.7402597402597403
95%CI-AUC = ('Binomial', 0.5275004861125896, 0.8932535525360982)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
Neg 1.00 1.00 1.00 56
Pos 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.56)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.02576767676767676
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----1-----
------test------
precision recall f1-score support
Neg 1.00 0.57 0.73 14
Pos 0.65 1.00 0.79 11
micro avg 0.76 0.76 0.76 25
macro avg 0.82 0.79 0.76 25
weighted avg 0.84 0.76 0.75 25
[Confusion Matrix]
TN FP
FN TP
[[ 8 6]
[ 0 11]]
cutoff = ('Youden', 0.3)
recall = 1.0
precision = 0.6470588235294118
sensitivity = 1.0
specificity = 0.5714285714285714
accuracy = 0.76
F1 = 0.7857142857142858
brier = 0.16261199999999998
AUC = 0.8506493506493505
95%CI-AUC = ('Binomial', 0.6518580242477625, 0.960255571076826)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
Neg 1.00 1.00 1.00 56
Pos 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.5)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.028577777777777785
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----2-----
------test------
precision recall f1-score support
Neg 0.77 0.71 0.74 14
Pos 0.67 0.73 0.70 11
micro avg 0.72 0.72 0.72 25
macro avg 0.72 0.72 0.72 25
weighted avg 0.72 0.72 0.72 25
[Confusion Matrix]
TN FP
FN TP
[[10 4]
[ 3 8]]
cutoff = ('Youden', 0.5)
recall = 0.7272727272727273
precision = 0.6666666666666666
sensitivity = 0.7272727272727273
specificity = 0.7142857142857143
accuracy = 0.72
F1 = 0.6956521739130435
brier = 0.20953600000000003
AUC = 0.737012987012987
95%CI-AUC = ('Binomial', 0.524048524156825, 0.8910439384661436)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
Neg 1.00 1.00 1.00 56
Pos 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.58)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.024962626262626266
AUC = 0.9999999999999999
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----3-----
------test------
precision recall f1-score support
Neg 0.86 0.86 0.86 14
Pos 0.82 0.82 0.82 11
micro avg 0.84 0.84 0.84 25
macro avg 0.84 0.84 0.84 25
weighted avg 0.84 0.84 0.84 25
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 2 9]]
cutoff = ('Youden', 0.4)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.15539199999999997
AUC = 0.8701298701298702
95%CI-AUC = ('Binomial', 0.6755464765688096, 0.9699559155493546)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
Neg 1.00 1.00 1.00 56
Pos 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.63)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.03036868686868688
AUC = 0.9999999999999999
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----4-----
------test------
precision recall f1-score support
Neg 0.86 0.86 0.86 14
Pos 0.80 0.80 0.80 10
micro avg 0.83 0.83 0.83 24
macro avg 0.83 0.83 0.83 24
weighted avg 0.83 0.83 0.83 24
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 2 8]]
cutoff = ('Youden', 0.55)
recall = 0.8
precision = 0.8
sensitivity = 0.8
specificity = 0.8571428571428571
accuracy = 0.8333333333333334
F1 = 0.8000000000000002
brier = 0.18628749999999997
AUC = 0.775
95%CI-AUC = ('Binomial', 0.5600187574967712, 0.918381272983291)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
Neg 1.00 1.00 1.00 56
Pos 1.00 1.00 1.00 44
micro avg 1.00 1.00 1.00 100
macro avg 1.00 1.00 1.00 100
weighted avg 1.00 1.00 1.00 100
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 44]]
cutoff = ('Youden', 0.61)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.024510999999999995
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9637833073548235, nan)
0 1 2 3 4 Mean \
recall 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
precision 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
sensitivity 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
specificity 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
accuracy 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
F1 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
brier 0.025768 0.028578 0.024963 0.030369 0.024511 0.026838
AUC 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Std
recall 0.000000e+00
precision 0.000000e+00
sensitivity 0.000000e+00
specificity 0.000000e+00
accuracy 0.000000e+00
F1 0.000000e+00
brier 2.528345e-03
AUC 7.850462e-17
0 1 2 3 4 Mean \
recall 0.818182 1.000000 0.727273 0.818182 0.800000 0.832727
precision 0.692308 0.647059 0.666667 0.818182 0.800000 0.724843
sensitivity 0.818182 1.000000 0.727273 0.818182 0.800000 0.832727
specificity 0.714286 0.571429 0.714286 0.857143 0.857143 0.742857
accuracy 0.760000 0.760000 0.720000 0.840000 0.833333 0.782667
F1 0.750000 0.785714 0.695652 0.818182 0.800000 0.769910
brier 0.204808 0.162612 0.209536 0.155392 0.186287 0.183727
AUC 0.740260 0.850649 0.737013 0.870130 0.775000 0.794610
Std
recall 0.100741
precision 0.078826
sensitivity 0.100741
specificity 0.119523
accuracy 0.051983
F1 0.048466
brier 0.024320
AUC 0.062249
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------all------
precision recall f1-score support
Neg 0.80 0.79 0.79 70
Pos 0.73 0.74 0.73 54
micro avg 0.77 0.77 0.77 124
macro avg 0.76 0.76 0.76 124
weighted avg 0.77 0.77 0.77 124
[Confusion Matrix]
TN FP
FN TP
[[55 15]
[14 40]]
cutoff = ('Youden', 0.5)
recall = 0.7407407407407407
precision = 0.7272727272727273
sensitivity = 0.7407407407407407
specificity = 0.7857142857142857
accuracy = 0.7661290322580645
F1 = 0.7339449541284404
brier = 0.18370645161290325
AUC = 0.7906084656084656
95%CI-AUC = ('Binomial', 0.7083768285246775, 0.8584686859802543)
-----Finish!-----
In [30]:
# set the name of dirs and names
fpath_out = 'TC'
name = 'TC'
# judge, read and standardize the X
if 'TC' in files:
X_TC_Sd = read_csv('TC/X_TC_Sd.csv',header=0,index_col=0)
else:
if not os.path.exists(fpath_out):os.makedirs(fpath_out)
X_TC = read_csv('X_all_TC.csv',header=0,index_col=0)
# X_TC_Sd = Std_features(X_TC)
# X_TC_Sd.to_csv('TC/X_TC_Sd.csv')
# set the structure of inputs
fpath_out = 'TC/LASSO'
if not os.path.exists(fpath_out):os.makedirs(fpath_out)
TC_input = paras(X=X_TC,y=y_all,ID_test=ID_test,ID_train=ID_train,name=name,fpath=fpath_out)
# the first step of feature selection
TC_output_Lasso = LASSO_CV.selection(TC_input,inner_n_splits=10)
plt.close('all')
# model developing
for clf,model_name in [(lr,'LR'),(svc,'SVM'),(rf,'RF')]:
TC_output_Lasso.model = clf
TC_output_Lasso.name = model_name
TC_output_Lasso.fpath = fpath_out
TC_results_CV,TC_result_all = ModelPred.OneCV(TC_output_Lasso,label=['NC','RB'])
plt.close()
-----Start LASSO_CV-----
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Alpha: 0.0360523386542392
Features reduced from 1029 to 25 by LASSO
Coefficients
Features
original_shape_Compactness2 -0.107429
wavelet-LHL_glcm_InverseVariance -0.071882
wavelet-LHH_firstorder_Kurtosis -0.068255
wavelet-LHH_firstorder_Mean -0.034603
exponential_glszm_SmallAreaLowGrayLevelEmphasis -0.030890
squareroot_glrlm_LongRunLowGrayLevelEmphasis -0.020861
wavelet-HHL_firstorder_Kurtosis -0.015594
exponential_glszm_SmallAreaEmphasis -0.013862
wavelet-HLL_glszm_SizeZoneNonUniformityNormalized -0.013301
wavelet-HHL_glcm_MaximumProbability -0.010520
wavelet-HLH_firstorder_Median -0.010065
wavelet-HHH_firstorder_Skewness -0.008420
wavelet-LHH_glcm_ClusterShade -0.001392
wavelet-LHH_firstorder_Skewness -0.001170
original_shape_Elongation 0.007603
wavelet-HLL_firstorder_Mean 0.007973
wavelet-LLH_glcm_Imc1 0.013875
exponential_firstorder_Minimum 0.022174
wavelet-HHL_glcm_InverseVariance 0.027059
wavelet-LLH_firstorder_Skewness 0.028777
wavelet-HHH_glcm_Autocorrelation 0.029876
wavelet-HHL_glszm_GrayLevelNonUniformity 0.050246
exponential_firstorder_Kurtosis 0.051480
original_shape_LeastAxis 0.096563
logarithm_glrlm_GrayLevelNonUniformity 0.130806
-----0-----
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Alpha: 0.04856092657583446
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Features reduced from 1029 to 19 by LASSO
Coefficients
Features
wavelet-LHL_glcm_InverseVariance -0.086180
original_shape_Compactness2 -0.040668
original_shape_Compactness1 -0.034836
wavelet-LHH_glcm_Correlation -0.032947
wavelet-HLL_glszm_SizeZoneNonUniformityNormalized -0.028016
wavelet-HHL_glcm_MaximumProbability -0.021003
wavelet-LHH_firstorder_Kurtosis -0.018609
wavelet-HHH_firstorder_Skewness -0.016327
wavelet-HHL_firstorder_Kurtosis -0.015393
exponential_glszm_SmallAreaHighGrayLevelEmphasis -0.010000
wavelet-LHH_firstorder_Skewness -0.006316
squareroot_glrlm_LongRunLowGrayLevelEmphasis -0.000322
square_glcm_Imc1 0.000115
wavelet-HLL_glcm_Imc1 0.004481
wavelet-HLH_firstorder_Mean 0.008077
logarithm_glszm_GrayLevelNonUniformity 0.031450
wavelet-LLH_firstorder_Skewness 0.034260
logarithm_glrlm_GrayLevelNonUniformity 0.101044
original_shape_SurfaceArea 0.112871
-----1-----
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Alpha: 0.03670206847993424
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Features reduced from 1029 to 27 by LASSO
Coefficients
Features
original_shape_Compactness2 -0.080054
wavelet-LHL_glcm_InverseVariance -0.060214
wavelet-HHH_firstorder_Skewness -0.033800
exponential_glszm_SmallAreaLowGrayLevelEmphasis -0.030710
wavelet-HLL_glszm_SizeZoneNonUniformityNormalized -0.026523
wavelet-LHH_glcm_ClusterShade -0.022398
exponential_glszm_SmallAreaEmphasis -0.022174
wavelet-HLL_firstorder_Kurtosis -0.021010
wavelet-LLH_firstorder_Median -0.011947
exponential_glcm_Correlation -0.011513
wavelet-LHH_firstorder_Skewness -0.007238
exponential_glszm_ZonePercentage -0.001331
wavelet-LHL_glszm_LargeAreaLowGrayLevelEmphasis 0.000271
wavelet-HLH_glrlm_ShortRunLowGrayLevelEmphasis 0.007248
logarithm_glszm_GrayLevelNonUniformity 0.008632
wavelet-LLH_glcm_Imc1 0.009854
wavelet-HLL_firstorder_Mean 0.015160
exponential_firstorder_Minimum 0.015915
wavelet-LHH_glszm_SizeZoneNonUniformity 0.015918
wavelet-LLH_glrlm_ShortRunLowGrayLevelEmphasis 0.016950
original_shape_Maximum2DDiameterRow 0.018271
wavelet-LLH_firstorder_Skewness 0.020378
wavelet-HLL_glrlm_ShortRunLowGrayLevelEmphasis 0.036538
square_glcm_InverseVariance 0.042734
wavelet-HHH_glszm_SizeZoneNonUniformity 0.069047
wavelet-HHH_firstorder_TotalEnergy 0.084055
logarithm_glrlm_GrayLevelNonUniformity 0.129956
-----2-----
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Alpha: 0.0746381877770226
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Features reduced from 1029 to 8 by LASSO
Coefficients
Features
wavelet-LHL_glcm_InverseVariance -0.061716
original_shape_Compactness2 -0.014791
wavelet-HHH_firstorder_Skewness -0.000515
wavelet-HHH_glszm_GrayLevelNonUniformity 0.008518
wavelet-LLH_firstorder_Skewness 0.014395
original_shape_Maximum2DDiameterRow 0.029537
original_shape_SurfaceArea 0.071065
logarithm_glrlm_GrayLevelNonUniformity 0.082379
-----3-----
Alpha: 0.054333567480365094
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems. ConvergenceWarning)
Features reduced from 1029 to 16 by LASSO
Coefficients
Features
original_shape_Compactness2 -0.072087
exponential_glszm_SmallAreaEmphasis -0.027201
wavelet-HHH_firstorder_Skewness -0.015654
exponential_glszm_SmallAreaLowGrayLevelEmphasis -0.008773
wavelet-HHL_glszm_SizeZoneNonUniformityNormalized -0.007408
wavelet-LHL_firstorder_Skewness -0.006026
squareroot_glrlm_LongRunLowGrayLevelEmphasis -0.003764
exponential_glszm_SmallAreaHighGrayLevelEmphasis -0.003733
wavelet-HLL_glcm_Imc1 0.000757
wavelet-LLH_firstorder_Skewness 0.006588
squareroot_glszm_LargeAreaHighGrayLevelEmphasis 0.007979
wavelet-HHH_glszm_SizeZoneNonUniformity 0.020410
wavelet-HHL_glszm_GrayLevelNonUniformity 0.028253
original_shape_Maximum2DDiameterRow 0.051761
original_shape_SurfaceArea 0.062984
logarithm_glrlm_GrayLevelNonUniformity 0.093593
-----4-----
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
NC 0.75 0.86 0.80 14
RB 0.78 0.64 0.70 11
micro avg 0.76 0.76 0.76 25
macro avg 0.76 0.75 0.75 25
weighted avg 0.76 0.76 0.76 25
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 4 7]]
cutoff = ('Youden', 0.6821680243603379)
recall = 0.6363636363636364
precision = 0.7777777777777778
sensitivity = 0.6363636363636364
specificity = 0.8571428571428571
accuracy = 0.76
F1 = 0.7000000000000001
brier = 0.22859117109200205
AUC = 0.7402597402597402
95%CI-AUC = ('Binomial', 0.5275004861125896, 0.8932535525360982)
------train------
precision recall f1-score support
NC 0.98 0.93 0.95 56
RB 0.91 0.98 0.94 43
micro avg 0.95 0.95 0.95 99
macro avg 0.95 0.95 0.95 99
weighted avg 0.95 0.95 0.95 99
[Confusion Matrix]
TN FP
FN TP
[[52 4]
[ 1 42]]
cutoff = ('Youden', 0.4474679752341732)
recall = 0.9767441860465116
precision = 0.9130434782608695
sensitivity = 0.9767441860465116
specificity = 0.9285714285714286
accuracy = 0.9494949494949495
F1 = 0.9438202247191011
brier = 0.06449347517994379
AUC = 0.9754983388704319
95%CI-AUC = ('Binomial', 0.922459034991654, 0.9960625121752468)
-----1-----
------test------
precision recall f1-score support
NC 0.89 0.57 0.70 14
RB 0.62 0.91 0.74 11
micro avg 0.72 0.72 0.72 25
macro avg 0.76 0.74 0.72 25
weighted avg 0.77 0.72 0.72 25
[Confusion Matrix]
TN FP
FN TP
[[ 8 6]
[ 1 10]]
cutoff = ('Youden', 0.4637770990546334)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.2797903889987711
AUC = 0.6688311688311688
95%CI-AUC = ('Binomial', 0.4537589122433224, 0.8422050155666465)
------train------
precision recall f1-score support
NC 0.89 0.96 0.92 56
RB 0.95 0.84 0.89 43
micro avg 0.91 0.91 0.91 99
macro avg 0.92 0.90 0.91 99
weighted avg 0.91 0.91 0.91 99
[Confusion Matrix]
TN FP
FN TP
[[54 2]
[ 7 36]]
cutoff = ('Youden', 0.6374783907804753)
recall = 0.8372093023255814
precision = 0.9473684210526315
sensitivity = 0.8372093023255814
specificity = 0.9642857142857143
accuracy = 0.9090909090909091
F1 = 0.8888888888888888
brier = 0.07214782964776685
AUC = 0.9717607973421927
95%CI-AUC = ('Binomial', 0.9169725085623216, 0.9945857476254419)
-----2-----
------test------
precision recall f1-score support
NC 0.65 0.93 0.76 14
RB 0.80 0.36 0.50 11
micro avg 0.68 0.68 0.68 25
macro avg 0.73 0.65 0.63 25
weighted avg 0.72 0.68 0.65 25
[Confusion Matrix]
TN FP
FN TP
[[13 1]
[ 7 4]]
cutoff = ('Youden', 0.9982237229853242)
recall = 0.36363636363636365
precision = 0.8
sensitivity = 0.36363636363636365
specificity = 0.9285714285714286
accuracy = 0.68
F1 = 0.5000000000000001
brier = 0.33880970084899076
AUC = 0.6233766233766234
95%CI-AUC = ('Binomial', 0.40903781261859157, 0.8073306742718196)
------train------
precision recall f1-score support
NC 0.96 0.96 0.96 56
RB 0.95 0.95 0.95 43
micro avg 0.96 0.96 0.96 99
macro avg 0.96 0.96 0.96 99
weighted avg 0.96 0.96 0.96 99
[Confusion Matrix]
TN FP
FN TP
[[54 2]
[ 2 41]]
cutoff = ('Youden', 0.5236519471783433)
recall = 0.9534883720930233
precision = 0.9534883720930233
sensitivity = 0.9534883720930233
specificity = 0.9642857142857143
accuracy = 0.9595959595959596
F1 = 0.9534883720930233
brier = 0.04856548130851752
AUC = 0.9883720930232558
95%CI-AUC = ('Binomial', 0.9424691550036365, 0.9995547707563381)
-----3-----
------test------
precision recall f1-score support
NC 0.86 0.86 0.86 14
RB 0.82 0.82 0.82 11
micro avg 0.84 0.84 0.84 25
macro avg 0.84 0.84 0.84 25
weighted avg 0.84 0.84 0.84 25
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 2 9]]
cutoff = ('Youden', 0.599799407845647)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.14674895879763852
AUC = 0.8441558441558441
95%CI-AUC = ('Binomial', 0.6441014717388924, 0.9568456190770979)
------train------
precision recall f1-score support
NC 0.81 0.86 0.83 56
RB 0.80 0.74 0.77 43
micro avg 0.81 0.81 0.81 99
macro avg 0.81 0.80 0.80 99
weighted avg 0.81 0.81 0.81 99
[Confusion Matrix]
TN FP
FN TP
[[48 8]
[11 32]]
cutoff = ('Youden', 0.4828510738214207)
recall = 0.7441860465116279
precision = 0.8
sensitivity = 0.7441860465116279
specificity = 0.8571428571428571
accuracy = 0.8080808080808081
F1 = 0.7710843373493975
brier = 0.14429198120010017
AUC = 0.8745847176079734
95%CI-AUC = ('Binomial', 0.7928642047132081, 0.9326333773965327)
-----4-----
------test------
precision recall f1-score support
NC 0.75 0.86 0.80 14
RB 0.75 0.60 0.67 10
micro avg 0.75 0.75 0.75 24
macro avg 0.75 0.73 0.73 24
weighted avg 0.75 0.75 0.74 24
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 4 6]]
cutoff = ('Youden', 0.5423370569996376)
recall = 0.6
precision = 0.75
sensitivity = 0.6
specificity = 0.8571428571428571
accuracy = 0.75
F1 = 0.6666666666666665
brier = 0.19652326626678018
AUC = 0.7785714285714286
95%CI-AUC = ('Binomial', 0.5639496858094944, 0.920619488150788)
------train------
precision recall f1-score support
NC 0.90 0.80 0.85 56
RB 0.78 0.89 0.83 44
micro avg 0.84 0.84 0.84 100
macro avg 0.84 0.84 0.84 100
weighted avg 0.85 0.84 0.84 100
[Confusion Matrix]
TN FP
FN TP
[[45 11]
[ 5 39]]
cutoff = ('Youden', 0.40576015564975926)
recall = 0.8863636363636364
precision = 0.78
sensitivity = 0.8863636363636364
specificity = 0.8035714285714286
accuracy = 0.84
F1 = 0.8297872340425532
brier = 0.11695831478320162
AUC = 0.9188311688311688
95%CI-AUC = ('Binomial', 0.8469797576538746, 0.9640441050455238)
0 1 2 3 4 Mean \
recall 0.976744 0.837209 0.953488 0.744186 0.886364 0.879598
precision 0.913043 0.947368 0.953488 0.800000 0.780000 0.878780
sensitivity 0.976744 0.837209 0.953488 0.744186 0.886364 0.879598
specificity 0.928571 0.964286 0.964286 0.857143 0.803571 0.903571
accuracy 0.949495 0.909091 0.959596 0.808081 0.840000 0.893253
F1 0.943820 0.888889 0.953488 0.771084 0.829787 0.877414
brier 0.064493 0.072148 0.048565 0.144292 0.116958 0.089291
AUC 0.975498 0.971761 0.988372 0.874585 0.918831 0.945809
Std
recall 0.093643
precision 0.082800
sensitivity 0.093643
specificity 0.070981
accuracy 0.066906
F1 0.077303
brier 0.039868
AUC 0.047873
0 1 2 3 4 Mean \
recall 0.636364 0.909091 0.363636 0.818182 0.600000 0.665455
precision 0.777778 0.625000 0.800000 0.818182 0.750000 0.754192
sensitivity 0.636364 0.909091 0.363636 0.818182 0.600000 0.665455
specificity 0.857143 0.571429 0.928571 0.857143 0.857143 0.814286
accuracy 0.760000 0.720000 0.680000 0.840000 0.750000 0.750000
F1 0.700000 0.740741 0.500000 0.818182 0.666667 0.685118
brier 0.228591 0.279790 0.338810 0.146749 0.196523 0.238093
AUC 0.740260 0.668831 0.623377 0.844156 0.778571 0.731039
Std
recall 0.211488
precision 0.076579
sensitivity 0.211488
specificity 0.139240
accuracy 0.059161
F1 0.117931
brier 0.074237
AUC 0.087465
------all------
precision recall f1-score support
NC 0.76 0.69 0.72 70
RB 0.64 0.72 0.68 54
micro avg 0.70 0.70 0.70 124
macro avg 0.70 0.70 0.70 124
weighted avg 0.71 0.70 0.70 124
[Confusion Matrix]
TN FP
FN TP
[[48 22]
[15 39]]
cutoff = ('Youden', 0.49102917595417156)
recall = 0.7222222222222222
precision = 0.639344262295082
sensitivity = 0.7222222222222222
specificity = 0.6857142857142857
accuracy = 0.7016129032258065
F1 = 0.6782608695652174
brier = 0.2384279345470789
AUC = 0.7227513227513228
95%CI-AUC = ('Binomial', 0.6352111815871254, 0.7993399038786299)
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
NC 0.76 0.93 0.84 14
RB 0.88 0.64 0.74 11
micro avg 0.80 0.80 0.80 25
macro avg 0.82 0.78 0.79 25
weighted avg 0.81 0.80 0.79 25
[Confusion Matrix]
TN FP
FN TP
[[13 1]
[ 4 7]]
cutoff = ('Youden', 0.7526627528175824)
recall = 0.6363636363636364
precision = 0.875
sensitivity = 0.6363636363636364
specificity = 0.9285714285714286
accuracy = 0.8
F1 = 0.7368421052631579
brier = 0.21298718003775144
AUC = 0.7532467532467533
95%CI-AUC = ('Binomial', 0.5414114365965182, 0.9019757858943704)
------train------
precision recall f1-score support
NC 0.98 0.93 0.95 56
RB 0.91 0.98 0.94 43
micro avg 0.95 0.95 0.95 99
macro avg 0.95 0.95 0.95 99
weighted avg 0.95 0.95 0.95 99
[Confusion Matrix]
TN FP
FN TP
[[52 4]
[ 1 42]]
cutoff = ('Youden', 0.42289476078803107)
recall = 0.9767441860465116
precision = 0.9130434782608695
sensitivity = 0.9767441860465116
specificity = 0.9285714285714286
accuracy = 0.9494949494949495
F1 = 0.9438202247191011
brier = 0.05113424740709731
AUC = 0.9842192691029901
95%CI-AUC = ('Binomial', 0.9357811252102437, 0.998764025229285)
-----1-----
------test------
precision recall f1-score support
NC 0.89 0.57 0.70 14
RB 0.62 0.91 0.74 11
micro avg 0.72 0.72 0.72 25
macro avg 0.76 0.74 0.72 25
weighted avg 0.77 0.72 0.72 25
[Confusion Matrix]
TN FP
FN TP
[[ 8 6]
[ 1 10]]
cutoff = ('Youden', 0.33078777850598057)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.2611122038633015
AUC = 0.6493506493506493
95%CI-AUC = ('Binomial', 0.43439476380998987, 0.8274697272569396)
------train------
precision recall f1-score support
NC 0.95 0.95 0.95 56
RB 0.93 0.93 0.93 43
micro avg 0.94 0.94 0.94 99
macro avg 0.94 0.94 0.94 99
weighted avg 0.94 0.94 0.94 99
[Confusion Matrix]
TN FP
FN TP
[[53 3]
[ 3 40]]
cutoff = ('Youden', 0.4130642527481867)
recall = 0.9302325581395349
precision = 0.9302325581395349
sensitivity = 0.9302325581395349
specificity = 0.9464285714285714
accuracy = 0.9393939393939394
F1 = 0.9302325581395349
brier = 0.053878309298437684
AUC = 0.9842192691029901
95%CI-AUC = ('Binomial', 0.9357811252102438, 0.998764025229285)
-----2-----
------test------
precision recall f1-score support
NC 0.86 0.43 0.57 14
RB 0.56 0.91 0.69 11
micro avg 0.64 0.64 0.64 25
macro avg 0.71 0.67 0.63 25
weighted avg 0.72 0.64 0.62 25
[Confusion Matrix]
TN FP
FN TP
[[ 6 8]
[ 1 10]]
cutoff = ('Youden', 0.3393315741265286)
recall = 0.9090909090909091
precision = 0.5555555555555556
sensitivity = 0.9090909090909091
specificity = 0.42857142857142855
accuracy = 0.64
F1 = 0.6896551724137931
brier = 0.2661181776241424
AUC = 0.6103896103896104
95%CI-AUC = ('Binomial', 0.39655178479998426, 0.7970580784279642)
------train------
precision recall f1-score support
NC 0.98 0.96 0.97 56
RB 0.95 0.98 0.97 43
micro avg 0.97 0.97 0.97 99
macro avg 0.97 0.97 0.97 99
weighted avg 0.97 0.97 0.97 99
[Confusion Matrix]
TN FP
FN TP
[[54 2]
[ 1 42]]
cutoff = ('Youden', 0.4579476842038234)
recall = 0.9767441860465116
precision = 0.9545454545454546
sensitivity = 0.9767441860465116
specificity = 0.9642857142857143
accuracy = 0.9696969696969697
F1 = 0.9655172413793104
brier = 0.039957808339794815
AUC = 0.9933554817275748
95%CI-AUC = ('Binomial', 0.9509283035284491, 0.9999683493757263)
-----3-----
------test------
precision recall f1-score support
NC 0.92 0.86 0.89 14
RB 0.83 0.91 0.87 11
micro avg 0.88 0.88 0.88 25
macro avg 0.88 0.88 0.88 25
weighted avg 0.88 0.88 0.88 25
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 1 10]]
cutoff = ('Youden', 0.486970763318792)
recall = 0.9090909090909091
precision = 0.8333333333333334
sensitivity = 0.9090909090909091
specificity = 0.8571428571428571
accuracy = 0.88
F1 = 0.8695652173913043
brier = 0.1413554023992835
AUC = 0.9025974025974025
95%CI-AUC = ('Binomial', 0.716645614645395, 0.9840107496477167)
------train------
precision recall f1-score support
NC 0.88 0.91 0.89 56
RB 0.88 0.84 0.86 43
micro avg 0.88 0.88 0.88 99
macro avg 0.88 0.87 0.88 99
weighted avg 0.88 0.88 0.88 99
[Confusion Matrix]
TN FP
FN TP
[[51 5]
[ 7 36]]
cutoff = ('Youden', 0.46817051105222773)
recall = 0.8372093023255814
precision = 0.8780487804878049
sensitivity = 0.8372093023255814
specificity = 0.9107142857142857
accuracy = 0.8787878787878788
F1 = 0.8571428571428572
brier = 0.12356101861703364
AUC = 0.9065614617940199
95%CI-AUC = ('Binomial', 0.8313204847049749, 0.9558256014013052)
-----4-----
------test------
precision recall f1-score support
NC 0.79 0.79 0.79 14
RB 0.70 0.70 0.70 10
micro avg 0.75 0.75 0.75 24
macro avg 0.74 0.74 0.74 24
weighted avg 0.75 0.75 0.75 24
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 3 7]]
cutoff = ('Youden', 0.5)
recall = 0.7
precision = 0.7
sensitivity = 0.7
specificity = 0.7857142857142857
accuracy = 0.75
F1 = 0.7
brier = 0.19791343338767742
AUC = 0.75
95%CI-AUC = ('Binomial', 0.5328871975773306, 0.9022695905254671)
------train------
precision recall f1-score support
NC 0.94 0.84 0.89 56
RB 0.82 0.93 0.87 44
micro avg 0.88 0.88 0.88 100
macro avg 0.88 0.89 0.88 100
weighted avg 0.89 0.88 0.88 100
[Confusion Matrix]
TN FP
FN TP
[[47 9]
[ 3 41]]
cutoff = ('Youden', 0.4221527777010454)
recall = 0.9318181818181818
precision = 0.82
sensitivity = 0.9318181818181818
specificity = 0.8392857142857143
accuracy = 0.88
F1 = 0.8723404255319149
brier = 0.10140142182316429
AUC = 0.953327922077922
95%CI-AUC = ('Binomial', 0.891636369344168, 0.9854339815642317)
0 1 2 3 4 Mean \
recall 0.976744 0.930233 0.976744 0.837209 0.931818 0.930550
precision 0.913043 0.930233 0.954545 0.878049 0.820000 0.899174
sensitivity 0.976744 0.930233 0.976744 0.837209 0.931818 0.930550
specificity 0.928571 0.946429 0.964286 0.910714 0.839286 0.917857
accuracy 0.949495 0.939394 0.969697 0.878788 0.880000 0.923475
F1 0.943820 0.930233 0.965517 0.857143 0.872340 0.913811
brier 0.051134 0.053878 0.039958 0.123561 0.101401 0.073987
AUC 0.984219 0.984219 0.993355 0.906561 0.953328 0.964337
Std
recall 0.056969
precision 0.052292
sensitivity 0.056969
specificity 0.048247
accuracy 0.041695
F1 0.046837
brier 0.036379
AUC 0.035679
0 1 2 3 4 Mean \
recall 0.636364 0.909091 0.909091 0.909091 0.700000 0.812727
precision 0.875000 0.625000 0.555556 0.833333 0.700000 0.717778
sensitivity 0.636364 0.909091 0.909091 0.909091 0.700000 0.812727
specificity 0.928571 0.571429 0.428571 0.857143 0.785714 0.714286
accuracy 0.800000 0.720000 0.640000 0.880000 0.750000 0.758000
F1 0.736842 0.740741 0.689655 0.869565 0.700000 0.747361
brier 0.212987 0.261112 0.266118 0.141355 0.197913 0.215897
AUC 0.753247 0.649351 0.610390 0.902597 0.750000 0.733117
Std
recall 0.133856
precision 0.135381
sensitivity 0.133856
specificity 0.208248
accuracy 0.089554
F1 0.071870
brier 0.051125
AUC 0.113460
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------all------
precision recall f1-score support
NC 0.79 0.70 0.74 70
RB 0.66 0.76 0.71 54
micro avg 0.73 0.73 0.73 124
macro avg 0.73 0.73 0.72 124
weighted avg 0.73 0.73 0.73 124
[Confusion Matrix]
TN FP
FN TP
[[49 21]
[13 41]]
cutoff = ('Youden', 0.486970763318792)
recall = 0.7592592592592593
precision = 0.6612903225806451
sensitivity = 0.7592592592592593
specificity = 0.7
accuracy = 0.7258064516129032
F1 = 0.7068965517241378
brier = 0.2160423104791631
AUC = 0.7293650793650793
95%CI-AUC = ('Binomial', 0.6422373802567326, 0.8052110565297395)
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
NC 0.85 0.79 0.81 14
RB 0.75 0.82 0.78 11
micro avg 0.80 0.80 0.80 25
macro avg 0.80 0.80 0.80 25
weighted avg 0.80 0.80 0.80 25
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 2 9]]
cutoff = ('Youden', 0.5)
recall = 0.8181818181818182
precision = 0.75
sensitivity = 0.8181818181818182
specificity = 0.7857142857142857
accuracy = 0.8
F1 = 0.7826086956521738
brier = 0.18852800000000003
AUC = 0.7987012987012987
95%CI-AUC = ('Binomial', 0.591497055915008, 0.9309022688169337)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
NC 1.00 1.00 1.00 56
RB 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.7)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.029481818181818185
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----1-----
------test------
precision recall f1-score support
NC 0.90 0.64 0.75 14
RB 0.67 0.91 0.77 11
micro avg 0.76 0.76 0.76 25
macro avg 0.78 0.78 0.76 25
weighted avg 0.80 0.76 0.76 25
[Confusion Matrix]
TN FP
FN TP
[[ 9 5]
[ 1 10]]
cutoff = ('Youden', 0.41)
recall = 0.9090909090909091
precision = 0.6666666666666666
sensitivity = 0.9090909090909091
specificity = 0.6428571428571429
accuracy = 0.76
F1 = 0.7692307692307692
brier = 0.22551200000000002
AUC = 0.7142857142857143
95%CI-AUC = ('Binomial', 0.5001623351600223, 0.8752665811796557)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
NC 1.00 1.00 1.00 56
RB 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.64)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.023317171717171715
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----2-----
------test------
precision recall f1-score support
NC 0.90 0.64 0.75 14
RB 0.67 0.91 0.77 11
micro avg 0.76 0.76 0.76 25
macro avg 0.78 0.78 0.76 25
weighted avg 0.80 0.76 0.76 25
[Confusion Matrix]
TN FP
FN TP
[[ 9 5]
[ 1 10]]
cutoff = ('Youden', 0.37)
recall = 0.9090909090909091
precision = 0.6666666666666666
sensitivity = 0.9090909090909091
specificity = 0.6428571428571429
accuracy = 0.76
F1 = 0.7692307692307692
brier = 0.24103600000000003
AUC = 0.6915584415584415
95%CI-AUC = ('Binomial', 0.47674078382928714, 0.8589755416055123)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
NC 1.00 1.00 1.00 56
RB 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.66)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.02397070707070707
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----3-----
------test------
precision recall f1-score support
NC 0.81 0.93 0.87 14
RB 0.89 0.73 0.80 11
micro avg 0.84 0.84 0.84 25
macro avg 0.85 0.83 0.83 25
weighted avg 0.85 0.84 0.84 25
[Confusion Matrix]
TN FP
FN TP
[[13 1]
[ 3 8]]
cutoff = ('Youden', 0.5)
recall = 0.7272727272727273
precision = 0.8888888888888888
sensitivity = 0.7272727272727273
specificity = 0.9285714285714286
accuracy = 0.84
F1 = 0.7999999999999999
brier = 0.15328799999999998
AUC = 0.8474025974025975
95%CI-AUC = ('Binomial', 0.6479714584992858, 0.9585609594568026)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
NC 1.00 1.00 1.00 56
RB 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.65)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.028382828282828287
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----4-----
------test------
precision recall f1-score support
NC 0.90 0.64 0.75 14
RB 0.64 0.90 0.75 10
micro avg 0.75 0.75 0.75 24
macro avg 0.77 0.77 0.75 24
weighted avg 0.79 0.75 0.75 24
[Confusion Matrix]
TN FP
FN TP
[[9 5]
[1 9]]
cutoff = ('Youden', 0.25)
recall = 0.9
precision = 0.6428571428571429
sensitivity = 0.9
specificity = 0.6428571428571429
accuracy = 0.75
F1 = 0.75
brier = 0.188325
AUC = 0.7642857142857142
95%CI-AUC = ('Binomial', 0.5483099022317911, 0.9115694038401689)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
NC 1.00 1.00 1.00 56
RB 1.00 1.00 1.00 44
micro avg 1.00 1.00 1.00 100
macro avg 1.00 1.00 1.00 100
weighted avg 1.00 1.00 1.00 100
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 44]]
cutoff = ('Youden', 0.63)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.025304000000000004
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9637833073548235, nan)
0 1 2 3 4 Mean \
recall 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
precision 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
sensitivity 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
specificity 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
accuracy 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
F1 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
brier 0.029482 0.023317 0.023971 0.028383 0.025304 0.026091
AUC 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Std
recall 0.000000
precision 0.000000
sensitivity 0.000000
specificity 0.000000
accuracy 0.000000
F1 0.000000
brier 0.002718
AUC 0.000000
0 1 2 3 4 Mean \
recall 0.818182 0.909091 0.909091 0.727273 0.900000 0.852727
precision 0.750000 0.666667 0.666667 0.888889 0.642857 0.723016
sensitivity 0.818182 0.909091 0.909091 0.727273 0.900000 0.852727
specificity 0.785714 0.642857 0.642857 0.928571 0.642857 0.728571
accuracy 0.800000 0.760000 0.760000 0.840000 0.750000 0.782000
F1 0.782609 0.769231 0.769231 0.800000 0.750000 0.774214
brier 0.188528 0.225512 0.241036 0.153288 0.188325 0.199338
AUC 0.798701 0.714286 0.691558 0.847403 0.764286 0.763247
Std
recall 0.079876
precision 0.101264
sensitivity 0.079876
specificity 0.127775
accuracy 0.037683
F1 0.018516
brier 0.034578
AUC 0.063001
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------all------
precision recall f1-score support
NC 0.79 0.79 0.79 70
RB 0.72 0.72 0.72 54
micro avg 0.76 0.76 0.76 124
macro avg 0.75 0.75 0.75 124
weighted avg 0.76 0.76 0.76 124
[Confusion Matrix]
TN FP
FN TP
[[55 15]
[15 39]]
cutoff = ('Youden', 0.5)
recall = 0.7222222222222222
precision = 0.7222222222222222
sensitivity = 0.7222222222222222
specificity = 0.7857142857142857
accuracy = 0.7580645161290323
F1 = 0.7222222222222222
brier = 0.1994266129032258
AUC = 0.7603174603174604
95%CI-AUC = ('Binomial', 0.675409312208698, 0.8323906547827852)
-----Finish!-----
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
In [31]:
# set the structure of inputs for further feature selection
fpath_out = 'TC/Forward_CV-0.85'
if not os.path.exists(fpath_out):os.makedirs(fpath_out)
TC_output_Lasso.typeof = Further_Selection.Forward_CV
TC_output_Lasso.model = lr
TC_output_Lasso.name = name
TC_output_Lasso.fpath = fpath_out
# the further feature selection
TC_output_Further = Further_Selection_CV.selection(TC_output_Lasso,n_splits=5,threshold=0.85)
plt.close('all')
# model developing
for clf,model_name in [(lr,'LR'),(svc,'SVM'),(rf,'RF')]:
TC_output_Further.model = clf
TC_output_Further.name = model_name
TC_output_Further.fpath = fpath_out
TC_results_CV,TC_result_all = ModelPred.OneCV(TC_output_Further,label=['NC','RB'])
plt.close()
-----Start Further_Selection_CV-----
-----0-----
-----1-----
-----2-----
-----3-----
-----4-----
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
NC 0.75 0.86 0.80 14
RB 0.78 0.64 0.70 11
micro avg 0.76 0.76 0.76 25
macro avg 0.76 0.75 0.75 25
weighted avg 0.76 0.76 0.76 25
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 4 7]]
cutoff = ('Youden', 0.5935665054636224)
recall = 0.6363636363636364
precision = 0.7777777777777778
sensitivity = 0.6363636363636364
specificity = 0.8571428571428571
accuracy = 0.76
F1 = 0.7000000000000001
brier = 0.23967355806862375
AUC = 0.6688311688311688
95%CI-AUC = ('Binomial', 0.4537589122433224, 0.8422050155666465)
------train------
precision recall f1-score support
NC 0.90 0.82 0.86 56
RB 0.79 0.88 0.84 43
micro avg 0.85 0.85 0.85 99
macro avg 0.85 0.85 0.85 99
weighted avg 0.85 0.85 0.85 99
[Confusion Matrix]
TN FP
FN TP
[[46 10]
[ 5 38]]
cutoff = ('Youden', 0.4570045746265311)
recall = 0.8837209302325582
precision = 0.7916666666666666
sensitivity = 0.8837209302325582
specificity = 0.8214285714285714
accuracy = 0.8484848484848485
F1 = 0.8351648351648352
brier = 0.12112781057578377
AUC = 0.9036544850498339
95%CI-AUC = ('Binomial', 0.8277595823694529, 0.9537895961783565)
-----1-----
------test------
precision recall f1-score support
NC 1.00 0.50 0.67 14
RB 0.61 1.00 0.76 11
micro avg 0.72 0.72 0.72 25
macro avg 0.81 0.75 0.71 25
weighted avg 0.83 0.72 0.71 25
[Confusion Matrix]
TN FP
FN TP
[[ 7 7]
[ 0 11]]
cutoff = ('Youden', 0.18430323714442437)
recall = 1.0
precision = 0.6111111111111112
sensitivity = 1.0
specificity = 0.5
accuracy = 0.72
F1 = 0.7586206896551725
brier = 0.23933088816911852
AUC = 0.7337662337662338
95%CI-AUC = ('Binomial', 0.5206066641883598, 0.8888229898052754)
------train------
precision recall f1-score support
NC 0.87 0.86 0.86 56
RB 0.82 0.84 0.83 43
micro avg 0.85 0.85 0.85 99
macro avg 0.85 0.85 0.85 99
weighted avg 0.85 0.85 0.85 99
[Confusion Matrix]
TN FP
FN TP
[[48 8]
[ 7 36]]
cutoff = ('Youden', 0.4692026255873121)
recall = 0.8372093023255814
precision = 0.8181818181818182
sensitivity = 0.8372093023255814
specificity = 0.8571428571428571
accuracy = 0.8484848484848485
F1 = 0.8275862068965518
brier = 0.1356711309972889
AUC = 0.8799833887043189
95%CI-AUC = ('Binomial', 0.7992560281640527, 0.9366604941047192)
-----2-----
------test------
precision recall f1-score support
NC 1.00 0.57 0.73 14
RB 0.65 1.00 0.79 11
micro avg 0.76 0.76 0.76 25
macro avg 0.82 0.79 0.76 25
weighted avg 0.84 0.76 0.75 25
[Confusion Matrix]
TN FP
FN TP
[[ 8 6]
[ 0 11]]
cutoff = ('Youden', 0.3325332929635006)
recall = 1.0
precision = 0.6470588235294118
sensitivity = 1.0
specificity = 0.5714285714285714
accuracy = 0.76
F1 = 0.7857142857142858
brier = 0.1909833301648981
AUC = 0.8051948051948052
95%CI-AUC = ('Binomial', 0.5988452401884774, 0.9348091837360017)
------train------
precision recall f1-score support
NC 0.84 0.86 0.85 56
RB 0.81 0.79 0.80 43
micro avg 0.83 0.83 0.83 99
macro avg 0.83 0.82 0.82 99
weighted avg 0.83 0.83 0.83 99
[Confusion Matrix]
TN FP
FN TP
[[48 8]
[ 9 34]]
cutoff = ('Youden', 0.45693721422475736)
recall = 0.7906976744186046
precision = 0.8095238095238095
sensitivity = 0.7906976744186046
specificity = 0.8571428571428571
accuracy = 0.8282828282828283
F1 = 0.8
brier = 0.14436349129027587
AUC = 0.8754152823920266
95%CI-AUC = ('Binomial', 0.793845177925711, 0.9332555462283807)
-----3-----
------test------
precision recall f1-score support
NC 0.86 0.86 0.86 14
RB 0.82 0.82 0.82 11
micro avg 0.84 0.84 0.84 25
macro avg 0.84 0.84 0.84 25
weighted avg 0.84 0.84 0.84 25
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 2 9]]
cutoff = ('Youden', 0.599799407845647)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.14674895879763852
AUC = 0.8441558441558441
95%CI-AUC = ('Binomial', 0.6441014717388924, 0.9568456190770979)
------train------
precision recall f1-score support
NC 0.81 0.86 0.83 56
RB 0.80 0.74 0.77 43
micro avg 0.81 0.81 0.81 99
macro avg 0.81 0.80 0.80 99
weighted avg 0.81 0.81 0.81 99
[Confusion Matrix]
TN FP
FN TP
[[48 8]
[11 32]]
cutoff = ('Youden', 0.4828510738214207)
recall = 0.7441860465116279
precision = 0.8
sensitivity = 0.7441860465116279
specificity = 0.8571428571428571
accuracy = 0.8080808080808081
F1 = 0.7710843373493975
brier = 0.14429198120010017
AUC = 0.8745847176079734
95%CI-AUC = ('Binomial', 0.7928642047132081, 0.9326333773965327)
-----4-----
------test------
precision recall f1-score support
NC 0.81 0.93 0.87 14
RB 0.88 0.70 0.78 10
micro avg 0.83 0.83 0.83 24
macro avg 0.84 0.81 0.82 24
weighted avg 0.84 0.83 0.83 24
[Confusion Matrix]
TN FP
FN TP
[[13 1]
[ 3 7]]
cutoff = ('Youden', 0.4896735575704285)
recall = 0.7
precision = 0.875
sensitivity = 0.7
specificity = 0.9285714285714286
accuracy = 0.8333333333333334
F1 = 0.7777777777777777
brier = 0.17676127157189805
AUC = 0.8
95%CI-AUC = ('Binomial', 0.5878433872065862, 0.9336879520344967)
------train------
precision recall f1-score support
NC 0.87 0.82 0.84 56
RB 0.79 0.84 0.81 44
micro avg 0.83 0.83 0.83 100
macro avg 0.83 0.83 0.83 100
weighted avg 0.83 0.83 0.83 100
[Confusion Matrix]
TN FP
FN TP
[[46 10]
[ 7 37]]
cutoff = ('Youden', 0.4150308181605387)
recall = 0.8409090909090909
precision = 0.7872340425531915
sensitivity = 0.8409090909090909
specificity = 0.8214285714285714
accuracy = 0.83
F1 = 0.8131868131868133
brier = 0.14801197153699552
AUC = 0.8741883116883117
95%CI-AUC = ('Binomial', 0.7928886657778024, 0.9320862729002128)
0 1 2 3 4 Mean \
recall 0.883721 0.837209 0.790698 0.744186 0.840909 0.819345
precision 0.791667 0.818182 0.809524 0.800000 0.787234 0.801321
sensitivity 0.883721 0.837209 0.790698 0.744186 0.840909 0.819345
specificity 0.821429 0.857143 0.857143 0.857143 0.821429 0.842857
accuracy 0.848485 0.848485 0.828283 0.808081 0.830000 0.832667
F1 0.835165 0.827586 0.800000 0.771084 0.813187 0.809404
brier 0.121128 0.135671 0.144363 0.144292 0.148012 0.138693
AUC 0.903654 0.879983 0.875415 0.874585 0.874188 0.881565
Std
recall 0.053381
precision 0.012698
sensitivity 0.053381
specificity 0.019562
accuracy 0.016817
F1 0.025325
brier 0.010817
AUC 0.012564
0 1 2 3 4 Mean \
recall 0.636364 1.000000 1.000000 0.818182 0.700000 0.830909
precision 0.777778 0.611111 0.647059 0.818182 0.875000 0.745826
sensitivity 0.636364 1.000000 1.000000 0.818182 0.700000 0.830909
specificity 0.857143 0.500000 0.571429 0.857143 0.928571 0.742857
accuracy 0.760000 0.720000 0.760000 0.840000 0.833333 0.782667
F1 0.700000 0.758621 0.785714 0.818182 0.777778 0.768059
brier 0.239674 0.239331 0.190983 0.146749 0.176761 0.198700
AUC 0.668831 0.733766 0.805195 0.844156 0.800000 0.770390
Std
recall 0.167579
precision 0.112744
sensitivity 0.167579
specificity 0.192989
accuracy 0.051983
F1 0.043702
brier 0.040526
AUC 0.069255
------all------
precision recall f1-score support
NC 0.78 0.77 0.78 70
RB 0.71 0.72 0.72 54
micro avg 0.75 0.75 0.75 124
macro avg 0.75 0.75 0.75 124
weighted avg 0.75 0.75 0.75 124
[Confusion Matrix]
TN FP
FN TP
[[54 16]
[15 39]]
cutoff = ('Youden', 0.4896735575704285)
recall = 0.7222222222222222
precision = 0.7090909090909091
sensitivity = 0.7222222222222222
specificity = 0.7714285714285715
accuracy = 0.75
F1 = 0.7155963302752293
brier = 0.19887652336881073
AUC = 0.7687830687830687
95%CI-AUC = ('Binomial', 0.6845699159150108, 0.8397337364041085)
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
NC 0.78 1.00 0.88 14
RB 1.00 0.64 0.78 11
micro avg 0.84 0.84 0.84 25
macro avg 0.89 0.82 0.83 25
weighted avg 0.88 0.84 0.83 25
[Confusion Matrix]
TN FP
FN TP
[[14 0]
[ 4 7]]
cutoff = ('Youden', 0.5960953444917525)
recall = 0.6363636363636364
precision = 1.0
sensitivity = 0.6363636363636364
specificity = 1.0
accuracy = 0.84
F1 = 0.7777777777777778
brier = 0.17933908693714762
AUC = 0.7792207792207791
95%CI-AUC = ('Binomial', 0.5697525091143241, 0.9188286472222847)
------train------
precision recall f1-score support
NC 0.91 0.93 0.92 56
RB 0.90 0.88 0.89 43
micro avg 0.91 0.91 0.91 99
macro avg 0.91 0.91 0.91 99
weighted avg 0.91 0.91 0.91 99
[Confusion Matrix]
TN FP
FN TP
[[52 4]
[ 5 38]]
cutoff = ('Youden', 0.5145698259956726)
recall = 0.8837209302325582
precision = 0.9047619047619048
sensitivity = 0.8837209302325582
specificity = 0.9285714285714286
accuracy = 0.9090909090909091
F1 = 0.8941176470588236
brier = 0.09517465706819654
AUC = 0.9472591362126246
95%CI-AUC = ('Binomial', 0.8830855404270885, 0.9821168222995504)
-----1-----
------test------
precision recall f1-score support
NC 0.89 0.57 0.70 14
RB 0.62 0.91 0.74 11
micro avg 0.72 0.72 0.72 25
macro avg 0.76 0.74 0.72 25
weighted avg 0.77 0.72 0.72 25
[Confusion Matrix]
TN FP
FN TP
[[ 8 6]
[ 1 10]]
cutoff = ('Youden', 0.28318586234196486)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.2184718045516427
AUC = 0.7207792207792207
95%CI-AUC = ('Binomial', 0.5069383185338913, 0.8798285008906752)
------train------
precision recall f1-score support
NC 0.91 0.88 0.89 56
RB 0.84 0.88 0.86 43
micro avg 0.88 0.88 0.88 99
macro avg 0.88 0.88 0.88 99
weighted avg 0.88 0.88 0.88 99
[Confusion Matrix]
TN FP
FN TP
[[49 7]
[ 5 38]]
cutoff = ('Youden', 0.4387797812177666)
recall = 0.8837209302325582
precision = 0.8444444444444444
sensitivity = 0.8837209302325582
specificity = 0.875
accuracy = 0.8787878787878788
F1 = 0.8636363636363636
brier = 0.1040112765268145
AUC = 0.9343853820598007
95%CI-AUC = ('Binomial', 0.8662562030644628, 0.9743361484128362)
-----2-----
------test------
precision recall f1-score support
NC 0.91 0.71 0.80 14
RB 0.71 0.91 0.80 11
micro avg 0.80 0.80 0.80 25
macro avg 0.81 0.81 0.80 25
weighted avg 0.82 0.80 0.80 25
[Confusion Matrix]
TN FP
FN TP
[[10 4]
[ 1 10]]
cutoff = ('Youden', 0.4305939014452084)
recall = 0.9090909090909091
precision = 0.7142857142857143
sensitivity = 0.9090909090909091
specificity = 0.7142857142857143
accuracy = 0.8
F1 = 0.8
brier = 0.19120369295528428
AUC = 0.8311688311688312
95%CI-AUC = ('Binomial', 0.6287807960115356, 0.9497868286347726)
------train------
precision recall f1-score support
NC 0.87 0.84 0.85 56
RB 0.80 0.84 0.82 43
micro avg 0.84 0.84 0.84 99
macro avg 0.84 0.84 0.84 99
weighted avg 0.84 0.84 0.84 99
[Confusion Matrix]
TN FP
FN TP
[[47 9]
[ 7 36]]
cutoff = ('Youden', 0.36881709710346)
recall = 0.8372093023255814
precision = 0.8
sensitivity = 0.8372093023255814
specificity = 0.8392857142857143
accuracy = 0.8383838383838383
F1 = 0.8181818181818183
brier = 0.13467755396317513
AUC = 0.8874584717607974
95%CI-AUC = ('Binomial', 0.8081692052211917, 0.9421672801219797)
-----3-----
------test------
precision recall f1-score support
NC 0.92 0.86 0.89 14
RB 0.83 0.91 0.87 11
micro avg 0.88 0.88 0.88 25
macro avg 0.88 0.88 0.88 25
weighted avg 0.88 0.88 0.88 25
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 1 10]]
cutoff = ('Youden', 0.4941039838830814)
recall = 0.9090909090909091
precision = 0.8333333333333334
sensitivity = 0.9090909090909091
specificity = 0.8571428571428571
accuracy = 0.88
F1 = 0.8695652173913043
brier = 0.13226585537413196
AUC = 0.9025974025974025
95%CI-AUC = ('Binomial', 0.716645614645395, 0.9840107496477167)
------train------
precision recall f1-score support
NC 0.88 0.91 0.89 56
RB 0.88 0.84 0.86 43
micro avg 0.88 0.88 0.88 99
macro avg 0.88 0.87 0.88 99
weighted avg 0.88 0.88 0.88 99
[Confusion Matrix]
TN FP
FN TP
[[51 5]
[ 7 36]]
cutoff = ('Youden', 0.4712907220618809)
recall = 0.8372093023255814
precision = 0.8780487804878049
sensitivity = 0.8372093023255814
specificity = 0.9107142857142857
accuracy = 0.8787878787878788
F1 = 0.8571428571428572
brier = 0.11566416773229307
AUC = 0.90656146179402
95%CI-AUC = ('Binomial', 0.8313204847049749, 0.9558256014013052)
-----4-----
------test------
precision recall f1-score support
NC 0.81 0.93 0.87 14
RB 0.88 0.70 0.78 10
micro avg 0.83 0.83 0.83 24
macro avg 0.84 0.81 0.82 24
weighted avg 0.84 0.83 0.83 24
[Confusion Matrix]
TN FP
FN TP
[[13 1]
[ 3 7]]
cutoff = ('Youden', 0.6188785849320634)
recall = 0.7
precision = 0.875
sensitivity = 0.7
specificity = 0.9285714285714286
accuracy = 0.8333333333333334
F1 = 0.7777777777777777
brier = 0.16030590494717958
AUC = 0.8857142857142857
95%CI-AUC = ('Binomial', 0.6898089360463363, 0.9781363807975156)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
NC 0.92 0.82 0.87 56
RB 0.80 0.91 0.85 44
micro avg 0.86 0.86 0.86 100
macro avg 0.86 0.87 0.86 100
weighted avg 0.87 0.86 0.86 100
[Confusion Matrix]
TN FP
FN TP
[[46 10]
[ 4 40]]
cutoff = ('Youden', 0.4515961376097948)
recall = 0.9090909090909091
precision = 0.8
sensitivity = 0.9090909090909091
specificity = 0.8214285714285714
accuracy = 0.86
F1 = 0.8510638297872342
brier = 0.11153252143106149
AUC = 0.9310064935064934
95%CI-AUC = ('Binomial', 0.8623654733618611, 0.9720401978261864)
0 1 2 3 4 Mean \
recall 0.883721 0.883721 0.837209 0.837209 0.909091 0.870190
precision 0.904762 0.844444 0.800000 0.878049 0.800000 0.845451
sensitivity 0.883721 0.883721 0.837209 0.837209 0.909091 0.870190
specificity 0.928571 0.875000 0.839286 0.910714 0.821429 0.875000
accuracy 0.909091 0.878788 0.838384 0.878788 0.860000 0.873010
F1 0.894118 0.863636 0.818182 0.857143 0.851064 0.856829
brier 0.095175 0.104011 0.134678 0.115664 0.111533 0.112212
AUC 0.947259 0.934385 0.887458 0.906561 0.931006 0.921334
Std
recall 0.031839
precision 0.046672
sensitivity 0.031839
specificity 0.045527
accuracy 0.026157
F1 0.027220
brier 0.014787
AUC 0.023988
0 1 2 3 4 Mean \
recall 0.636364 0.909091 0.909091 0.909091 0.700000 0.812727
precision 1.000000 0.625000 0.714286 0.833333 0.875000 0.809524
sensitivity 0.636364 0.909091 0.909091 0.909091 0.700000 0.812727
specificity 1.000000 0.571429 0.714286 0.857143 0.928571 0.814286
accuracy 0.840000 0.720000 0.800000 0.880000 0.833333 0.814667
F1 0.777778 0.740741 0.800000 0.869565 0.777778 0.793172
brier 0.179339 0.218472 0.191204 0.132266 0.160306 0.176317
AUC 0.779221 0.720779 0.831169 0.902597 0.885714 0.823896
Std
recall 0.133856
precision 0.145133
sensitivity 0.133856
specificity 0.172023
accuracy 0.060074
F1 0.047711
brier 0.032422
AUC 0.075328
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------all------
precision recall f1-score support
NC 0.76 0.86 0.81 70
RB 0.78 0.65 0.71 54
micro avg 0.77 0.77 0.77 124
macro avg 0.77 0.75 0.76 124
weighted avg 0.77 0.77 0.76 124
[Confusion Matrix]
TN FP
FN TP
[[60 10]
[19 35]]
cutoff = ('Youden', 0.5960953444917525)
recall = 0.6481481481481481
precision = 0.7777777777777778
sensitivity = 0.6481481481481481
specificity = 0.8571428571428571
accuracy = 0.7661290322580645
F1 = 0.707070707070707
brier = 0.1764463928563506
AUC = 0.8103174603174603
95%CI-AUC = ('Binomial', 0.730129771117034, 0.8751222495948888)
-----Finish!-----
-----Start Model-----
-----0-----
------test------
precision recall f1-score support
NC 0.86 0.86 0.86 14
RB 0.82 0.82 0.82 11
micro avg 0.84 0.84 0.84 25
macro avg 0.84 0.84 0.84 25
weighted avg 0.84 0.84 0.84 25
[Confusion Matrix]
TN FP
FN TP
[[12 2]
[ 2 9]]
cutoff = ('Youden', 0.56)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.17494800000000002
AUC = 0.8116883116883118
95%CI-AUC = ('Binomial', 0.6062459490380402, 0.9386536573797263)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
NC 1.00 1.00 1.00 56
RB 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.64)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.0269020202020202
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----1-----
------test------
precision recall f1-score support
NC 0.79 0.79 0.79 14
RB 0.73 0.73 0.73 11
micro avg 0.76 0.76 0.76 25
macro avg 0.76 0.76 0.76 25
weighted avg 0.76 0.76 0.76 25
[Confusion Matrix]
TN FP
FN TP
[[11 3]
[ 3 8]]
cutoff = ('Youden', 0.57)
recall = 0.7272727272727273
precision = 0.7272727272727273
sensitivity = 0.7272727272727273
specificity = 0.7857142857142857
accuracy = 0.76
F1 = 0.7272727272727273
brier = 0.195104
AUC = 0.7792207792207793
95%CI-AUC = ('Binomial', 0.569752509114324, 0.9188286472222847)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------train------
precision recall f1-score support
NC 1.00 1.00 1.00 56
RB 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.59)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.022625252525252524
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
-----2-----
------test------
precision recall f1-score support
NC 0.89 0.57 0.70 14
RB 0.62 0.91 0.74 11
micro avg 0.72 0.72 0.72 25
macro avg 0.76 0.74 0.72 25
weighted avg 0.77 0.72 0.72 25
[Confusion Matrix]
TN FP
FN TP
[[ 8 6]
[ 1 10]]
cutoff = ('Youden', 0.4)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.183032
AUC = 0.8084415584415585
95%CI-AUC = ('Binomial', 0.6025389264488501, 0.9367393742358567)
------train------
precision recall f1-score support
NC 1.00 1.00 1.00 56
RB 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.58)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.026202020202020206
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
-----3-----
------test------
precision recall f1-score support
NC 0.82 1.00 0.90 14
RB 1.00 0.73 0.84 11
micro avg 0.88 0.88 0.88 25
macro avg 0.91 0.86 0.87 25
weighted avg 0.90 0.88 0.88 25
[Confusion Matrix]
TN FP
FN TP
[[14 0]
[ 3 8]]
cutoff = ('Youden', 0.5)
recall = 0.7272727272727273
precision = 1.0
sensitivity = 0.7272727272727273
specificity = 1.0
accuracy = 0.88
F1 = 0.8421052631578948
brier = 0.157724
AUC = 0.827922077922078
95%CI-AUC = ('Binomial', 0.6249889329318279, 0.9479750440455293)
------train------
precision recall f1-score support
NC 1.00 1.00 1.00 56
RB 1.00 1.00 1.00 43
micro avg 1.00 1.00 1.00 99
macro avg 1.00 1.00 1.00 99
weighted avg 1.00 1.00 1.00 99
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 43]]
cutoff = ('Youden', 0.6)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.026823232323232322
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
-----4-----
------test------
precision recall f1-score support
NC 0.91 0.71 0.80 14
RB 0.69 0.90 0.78 10
micro avg 0.79 0.79 0.79 24
macro avg 0.80 0.81 0.79 24
weighted avg 0.82 0.79 0.79 24
[Confusion Matrix]
TN FP
FN TP
[[10 4]
[ 1 9]]
cutoff = ('Youden', 0.06)
recall = 0.9
precision = 0.6923076923076923
sensitivity = 0.9
specificity = 0.7142857142857143
accuracy = 0.7916666666666666
F1 = 0.7826086956521738
brier = 0.17835416666666667
AUC = 0.8535714285714286
95%CI-AUC = ('Binomial', 0.6501954948532898, 0.9632044978249255)
------train------
precision recall f1-score support
NC 1.00 1.00 1.00 56
RB 1.00 1.00 1.00 44
micro avg 1.00 1.00 1.00 100
macro avg 1.00 1.00 1.00 100
weighted avg 1.00 1.00 1.00 100
[Confusion Matrix]
TN FP
FN TP
[[56 0]
[ 0 44]]
cutoff = ('Youden', 0.63)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.02742300000000001
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9637833073548235, nan)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
0 1 2 3 4 Mean \
recall 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
precision 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
sensitivity 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
specificity 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
accuracy 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
F1 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
brier 0.026902 0.022625 0.026202 0.026823 0.027423 0.025995
AUC 1.000000 1.000000 1.000000 1.000000 1.000000 1.000000
Std
recall 0.000000
precision 0.000000
sensitivity 0.000000
specificity 0.000000
accuracy 0.000000
F1 0.000000
brier 0.001933
AUC 0.000000
0 1 2 3 4 Mean \
recall 0.818182 0.727273 0.909091 0.727273 0.900000 0.816364
precision 0.818182 0.727273 0.625000 1.000000 0.692308 0.772552
sensitivity 0.818182 0.727273 0.909091 0.727273 0.900000 0.816364
specificity 0.857143 0.785714 0.571429 1.000000 0.714286 0.785714
accuracy 0.840000 0.760000 0.720000 0.880000 0.791667 0.798333
F1 0.818182 0.727273 0.740741 0.842105 0.782609 0.782182
brier 0.174948 0.195104 0.183032 0.157724 0.178354 0.177832
AUC 0.811688 0.779221 0.808442 0.827922 0.853571 0.816169
Std
recall 0.088700
precision 0.144979
sensitivity 0.088700
specificity 0.159719
accuracy 0.063355
F1 0.049039
brier 0.013586
AUC 0.027304
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning) /home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). max_open_warning, RuntimeWarning)
------all------
precision recall f1-score support
NC 0.78 0.87 0.82 70
RB 0.80 0.69 0.74 54
micro avg 0.79 0.79 0.79 124
macro avg 0.79 0.78 0.78 124
weighted avg 0.79 0.79 0.79 124
[Confusion Matrix]
TN FP
FN TP
[[61 9]
[17 37]]
cutoff = ('Youden', 0.56)
recall = 0.6851851851851852
precision = 0.8043478260869565
sensitivity = 0.6851851851851852
specificity = 0.8714285714285714
accuracy = 0.7903225806451613
F1 = 0.74
brier = 0.1778282258064516
AUC = 0.8072751322751323
95%CI-AUC = ('Binomial', 0.7267550905482664, 0.8725691357364078)
-----Finish!-----
In [ ]: